claude-mem-lite 3.7.0 → 3.7.1

package/lib/doctor-benchmark.mjs

@@ -16,7 +16,7 @@ import { sanitizeFtsQuery, OBS_BM25 } from '../utils.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const SERVER_PATH = join(__dirname, '..', 'server.mjs');
-const SERVER_INTERNALS_PATH = join(__dirname, '..', 'server-internals.mjs');
+const SEARCH_SCORING_PATH = join(__dirname, '..', 'search-scoring.mjs');
 const BENCHMARK_VERSION = '1';
 
 function extractStringArrayBody(body) {
@@ -40,7 +40,7 @@ function extractStringArrayBody(body) {
  *   1. template literal in server.mjs:  instructions: `...`
  *   2. array-join in server.mjs:        instructions: [ '...', '...' ].join('\n')
  *   3. (v2.31.3+) builder call in server.mjs referencing INSTRUCTIONS_BASE +
- *      INSTRUCTIONS_VERBOSE arrays in server-internals.mjs. Measured at the
+ *      INSTRUCTIONS_VERBOSE arrays in search-scoring.mjs. Measured at the
  *      verbose form — this is the cost-per-turn baseline the benchmark tracks.
  * Returns '' if no shape matches (caller treats byte count as 0).
  */
@@ -56,10 +56,10 @@ function readMcpInstructions() {
   if (arr) return extractStringArrayBody(arr[1]).join('\n');
 
   // Form 3: buildServerInstructions() — reconstruct verbose form from
-  // server-internals.mjs INSTRUCTIONS_BASE + INSTRUCTIONS_VERBOSE arrays.
+  // search-scoring.mjs INSTRUCTIONS_BASE + INSTRUCTIONS_VERBOSE arrays.
   if (/instructions:\s*buildServerInstructions\(/.test(src)) {
     let internals;
-    try { internals = readFileSync(SERVER_INTERNALS_PATH, 'utf8'); } catch { return ''; }
+    try { internals = readFileSync(SEARCH_SCORING_PATH, 'utf8'); } catch { return ''; }
     const base = internals.match(/INSTRUCTIONS_BASE\s*=\s*\[([\s\S]*?)\];/);
     const verbose = internals.match(/INSTRUCTIONS_VERBOSE\s*=\s*\[([\s\S]*?)\];/);
     const parts = [];

package/lib/search-core.mjs

@@ -1,26 +1,33 @@
-// Shared cross-source search core (query build / source queries / scoring
-// normalization / sort / pagination math).
+// Shared cross-source search core for cmdSearch (CLI) and mem_search (MCP).
 //
-// Single source of truth for cmdSearch (CLI) and mem_search (MCP). The
-// observation path already converged in search-engine.mjs (#8198/#8212); the
-// sessions/prompts FTS queries, CJK precision + LIKE fallback, cross-source
-// score normalization, user-sort, over-fetch sizing, and date-bound parsing
-// were still copy-pasted and synced by "paired-path" comments — the drift
-// class compress-core (ARCH-1), recall-core, and timeline-core were extracted
-// to close. Call sites keep what legitimately differs: flag/schema parsing,
-// result-row dialect (CLI `_source`+raw columns vs MCP `source`+mapped
-// fields), error-message wording, and output rendering.
+// coreRunSearchPipeline (below) is the SINGLE orchestration body — deep /
+// auto-escalation → per-source query (obs hybrid + sessions + prompts) →
+// cross-source normalize+sort → context re-rank + supersede → tier filter →
+// user sort → count+paginate. The two ~180-line bodies that cmdSearch and
+// runSearchPipeline used to keep hand-synced via "paired-path" comments are
+// gone (audit P1-2); each surface is now a thin adapter that parses/validates
+// in and renders out. Cross-surface equivalence is enforced by
+// tests/search-parity.test.mjs, not by comments.
 //
-// Behavioral asymmetries that are PRESERVED, not converged (documented so a
-// future "fix" is a deliberate contract change, not an accident):
-//   • CLI forces source=observations when --type/--tier/--importance/--branch
-//     is set; MCP only forces it for obs_type.
+// Surfaces legitimately differ on a few policy points; each is an explicit opt
+// on coreRunSearchPipeline (obsTypeFallback / crossSourceEpochSortNoFts /
+// rerankPolicy / recentListingNoFts / tolerateMissingFts / tierPosition …) so
+// behavior is strictly preserved and any future convergence is a deliberate
+// change, not an accident. Notable preserved asymmetries:
+//   • CLI forces source=observations for --type/--tier/--importance/--branch;
+//     MCP only forces it for obs_type. (effectiveSource is computed per-adapter.)
 //   • CLI warns on inverted --from/--to ranges; MCP does not.
-//   • CLI wraps session/prompt FTS in try/catch for pre-FTS legacy DBs.
+//   • CLI tolerates missing session/prompt FTS (pre-FTS legacy DBs); MCP does not.
+//   • MCP lists-recent-by-type on a 0-match obs_type query; CLI does not (#8217).
+//
+// Result rows use one canonical `source` key; session/prompt rows carry dual
+// keys (date=created_at, text=prompt_text, session=content_session_id) so each
+// surface's renderer reads its own field names off a single row shape.
 
 import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, DEFAULT_DECAY_HALF_LIFE_MS } from '../utils.mjs';
 import { cjkPrecisionOk, extractCjkLikePatterns } from '../nlp.mjs';
 import { computeTier } from '../tier.mjs';
+import { countSearchTotal, attachBodyTokens } from '../search-engine.mjs';
 
 /** Sanitize a user query to FTS5 syntax; optionally force OR semantics. */
 export function buildSearchFtsQuery(query, { or = false } = {}) {
@@ -198,3 +205,252 @@ export function applyTierFilter(db, results, { tier, sourceKey, currentProject }
     return full && computeTier(full, tierCtx) === tier;
   });
 }
+
+/**
+ * Finalize a merged, scored result set into one page: compute the TRUE
+ * (limit/offset-invariant) population, slice the requested page, and attach the
+ * ~Nt fetch-cost hint. Single source of truth for the count+paginate+enrich tail
+ * of cmdSearch (CLI) and runSearchPipeline (MCP) — exactly where #8635 drifted
+ * (the over-fetch cap leaked into the reported total on BOTH sides independently).
+ *
+ * `total`: every source over-fetched from offset 0 (computePerSourceWindow), so
+ * results.length is the over-fetched candidate pool, NOT the population —
+ * countSearchTotal re-derives the real MATCH+filter count. Clamp to
+ * >= results.length so vector/concept-augmented obs rows are never undercounted
+ * (#8217/#8638). For deep (explicit or auto-escalated) the population IS the fused
+ * variant set already in `results` (deepSearch is obs-only, capped at
+ * perSourceLimit); countSearchTotal would instead count the ORIGINAL query's FTS
+ * matches — wrong, and ~0 on the vocabulary-mismatch queries deep exists for (F1).
+ *
+ * Pagination always slices: single-source results can exceed SQL LIMIT via
+ * expansion (concept co-occurrence / PRF / vector), and `offset` is applied
+ * exactly ONCE here (the per-source SQL always saw offset 0).
+ *
+ * @returns {{ total: number, page: object[] }}
+ */
+export function finalizeSearchPage(db, results, {
+  isDeep, offset, limit, effectiveSource, ftsQuery, orFallbackFired,
+  project = null, obsType = null, importance = null, branch = null,
+  epochFrom = null, epochTo = null, includeNoise = false,
+}) {
+  const total = isDeep
+    ? results.length
+    : Math.max(countSearchTotal(db, {
+      effectiveSource: effectiveSource || null,
+      ftsQuery,
+      obsFtsQuery: effectiveObsFtsQuery(ftsQuery, orFallbackFired),
+      args: { project: project || null, obs_type: obsType || null, importance: importance || null, branch: branch || null },
+      project: project || null,
+      epochFrom, epochTo, includeNoise,
+    }), results.length);
+  const page = results.slice(offset, offset + limit);
+  attachBodyTokens(db, page);
+  return { total, page };
+}
+
+/**
+ * Unified cross-source search orchestrator — single source of truth for the CLI
+ * (cmdSearch) and MCP (mem_search) search bodies: deep / auto-escalation →
+ * per-source query (obs hybrid + sessions + prompts) → cross-source
+ * normalize+sort → context re-rank + supersede → tier post-filter → user sort →
+ * count+paginate. The two surfaces legitimately differ on a handful of policy
+ * points; each is a named opt so a future "fix" is a deliberate contract change,
+ * not an accident (see the per-opt comments).
+ *
+ * The core does NO flag/schema parsing, NO stdout/stderr, NO formatting — adapters
+ * validate+parse on the way in and render on the way out. Session/prompt rows
+ * carry dual keys (`date`=created_at, `text`=prompt_text, `session`=content_session_id)
+ * so both renderers read their own field names off one canonical row.
+ *
+ * #8743: `db` comes ONLY from `ctx.db` — there is no module-global fallback, so a
+ * per-source leg can never silently query the wrong database.
+ *
+ * @returns {Promise<{ page:object[], total:number, preFinalizeCount:number, isDeep:boolean,
+ *   escalated:boolean, escalatedObsCount:number, variants:object[]|null,
+ *   reranked:boolean, orFallbackFired:boolean, effectiveSource:string|null, ftsQuery:string }>}
+ */
+export async function coreRunSearchPipeline(ctx, opts) {
+  const {
+    db, currentProject = null, env = process.env,
+    searchObservationsHybrid, deepSearch, shouldEscalateToDeep,
+    autoDeepLlmReady, reRankWithContext, markSuperseded,
+    llm = null, rerankLlm = undefined,
+  } = ctx;
+  const {
+    query, ftsQuery, effectiveSource = null, deepMode = 'normal', rerank = false,
+    limit, offset, project = null, obsType = null, importance = null, branch = null,
+    includeNoise = false, epochFrom = null, epochTo = null, sort = 'relevance', tier = null,
+    // ── surface policy (strict behavior-preservation; the two surfaces differ) ──
+    obsTypeFallback = false,           // A5: list-recent-by-type when 0 matches — MCP true, CLI false (#8217 removed it from CLI)
+    crossSourceEpochSortNoFts = false, // A3: epoch-sort the cross-source set when no ftsQuery — MCP true, CLI false
+    rerankPolicy = 'mcp',              // A4: re-rank/supersede gate + re-sort condition — 'mcp' | 'cli'
+    rerankProject = null,              // reRankWithContext project — MCP currentProject, CLI project||inferProject()
+    recentListingNoFts = false,        // session/prompt recent-listing when no ftsQuery (explicit --source) — MCP true, CLI false
+    tolerateMissingFts = false,        // wrap session/prompt FTS in try/catch for pre-FTS legacy DBs — CLI true, MCP false
+    tierPosition = 'late',             // tier filter vs re-rank ordering — MCP 'late' (after re-rank), CLI 'early' (in obs block)
+    tierProject = null,                // applyTierFilter project — MCP project||currentProject, CLI project||inferProject()
+  } = opts;
+
+  const { perSourceLimit, perSourceOffset } = computePerSourceWindow(limit, offset);
+  const isCrossSource = !effectiveSource;
+  const results = [];
+  let orFallbackFired = false;
+  let deepVariants = null;
+  let deepReranked = false;
+  let isDeep = deepMode === 'deep';
+  let escalated = false;
+  let escalatedObsCount = 0;
+
+  const obsCtx = {
+    db, ftsQuery,
+    args: { project, obs_type: obsType, importance, branch, include_noise: includeNoise },
+    epochFrom, epochTo, perSourceLimit, perSourceOffset, currentProject, limit,
+    orFallbackFired: false,
+  };
+
+  const runDeep = async ({ auto = false } = {}) => {
+    const ds = await deepSearch(db, {
+      query, project, type: obsType, importance, branch, includeNoise,
+      epochFrom, epochTo, limit: perSourceLimit, currentProject,
+    }, llm ? { llm, rerank: rerank && !auto, rerankLlm } : { auto, rerank: rerank && !auto, rerankLlm });
+    deepVariants = ds.variants;
+    deepReranked = ds.reranked;
+    return ds.results;
+  };
+
+  // ── Observations (hybrid engine; deep / auto-escalation; obs rows already carry source:'obs') ──
+  if (!effectiveSource || effectiveSource === 'observations') {
+    if (deepMode === 'deep') {
+      results.push(...await runDeep());
+    } else {
+      results.push(...searchObservationsHybrid(db, obsCtx));
+      if (obsCtx.orFallbackFired) orFallbackFired = true;
+      const obsCountBefore = results.filter((r) => r.source === 'obs').length;
+      if (deepMode === 'auto' && autoDeepLlmReady(env, llm) &&
+          shouldEscalateToDeep(results.filter((r) => r.source === 'obs'), obsCtx, { db, project })) {
+        const deepRows = await runDeep({ auto: true });
+        results.length = 0;
+        results.push(...deepRows);
+        isDeep = true;
+        escalated = true;
+        escalatedObsCount = obsCountBefore;
+      }
+    }
+  }
+
+  // ── Tier post-filter, CLI position: obs-only (tier forces observations), before re-rank ──
+  if (tier && tierPosition === 'early') {
+    const filtered = applyTierFilter(db, results, { tier, sourceKey: 'source', currentProject: tierProject });
+    results.length = 0; results.push(...filtered);
+  }
+
+  // ── Sessions (FTS via shared helper; optional recent-listing when no ftsQuery) ──
+  if ((!effectiveSource || effectiveSource === 'sessions') && !isDeep) {
+    const pushSessions = () => {
+      if (ftsQuery) {
+        const rows = searchSessionsFts(db, { ftsQuery, project, projectBoost: project ? null : currentProject, epochFrom, epochTo, perSourceLimit, perSourceOffset });
+        for (const r of rows) results.push({ ...r, source: 'session', date: r.created_at });
+      } else if (recentListingNoFts && effectiveSource === 'sessions') {
+        const params = []; const wheres = [];
+        if (project) { wheres.push('project = ?'); params.push(project); }
+        if (epochFrom !== null) { wheres.push('created_at_epoch >= ?'); params.push(epochFrom); }
+        if (epochTo !== null) { wheres.push('created_at_epoch <= ?'); params.push(epochTo); }
+        const where = wheres.length ? `WHERE ${wheres.join(' AND ')}` : '';
+        params.push(perSourceLimit, perSourceOffset);
+        const rows = db.prepare(`
+          SELECT id, request, completed, project, created_at, created_at_epoch
+          FROM session_summaries ${where}
+          ORDER BY created_at_epoch DESC
+          LIMIT ? OFFSET ?
+        `).all(...params);
+        for (const r of rows) results.push({ ...r, source: 'session', date: r.created_at });
+      }
+    };
+    if (tolerateMissingFts) { try { pushSessions(); } catch { /* session FTS may not exist in older DBs */ } } else pushSessions();
+  }
+
+  // ── Prompts (FTS via shared helper incl. CJK gate; optional recent-listing) ──
+  if ((!effectiveSource || effectiveSource === 'prompts') && !isDeep) {
+    const pushPrompts = () => {
+      if (ftsQuery) {
+        const rows = searchPromptsFts(db, { query, ftsQuery, project, epochFrom, epochTo, perSourceLimit, perSourceOffset });
+        for (const r of rows) results.push({ ...r, source: 'prompt', date: r.created_at, text: r.prompt_text, session: r.content_session_id });
+      } else if (recentListingNoFts && effectiveSource === 'prompts') {
+        const params = []; const wheres = [];
+        if (project) { wheres.push('s.project = ?'); params.push(project); }
+        if (epochFrom !== null) { wheres.push('p.created_at_epoch >= ?'); params.push(epochFrom); }
+        if (epochTo !== null) { wheres.push('p.created_at_epoch <= ?'); params.push(epochTo); }
+        const where = wheres.length ? `WHERE ${wheres.join(' AND ')}` : '';
+        params.push(perSourceLimit, perSourceOffset);
+        const rows = db.prepare(`
+          SELECT p.id, p.prompt_text, p.content_session_id, p.created_at, p.created_at_epoch
+          FROM user_prompts p
+          JOIN sdk_sessions s ON p.content_session_id = s.content_session_id
+          ${where}
+          ORDER BY p.created_at_epoch DESC
+          LIMIT ? OFFSET ?
+        `).all(...params);
+        for (const r of rows) results.push({ ...r, source: 'prompt', date: r.created_at, text: r.prompt_text, session: r.content_session_id });
+      }
+    };
+    if (tolerateMissingFts) { try { pushPrompts(); } catch { /* prompt FTS may not exist in older DBs */ } } else pushPrompts();
+  }
+
+  // ── Type-list fallback (MCP): obs_type set + 0 matches → list recent of that type ──
+  if (obsTypeFallback && results.length === 0 && obsType) {
+    const typeWheres = ['COALESCE(compressed_into, 0) = 0', 'superseded_at IS NULL', 'type = ?'];
+    const typeParams = [obsType];
+    if (project) { typeWheres.push('project = ?'); typeParams.push(project); }
+    if (epochFrom !== null) { typeWheres.push('created_at_epoch >= ?'); typeParams.push(epochFrom); }
+    if (epochTo !== null) { typeWheres.push('created_at_epoch <= ?'); typeParams.push(epochTo); }
+    if (importance) { typeWheres.push('COALESCE(importance, 1) >= ?'); typeParams.push(importance); }
+    typeParams.push(limit);
+    const typeRows = db.prepare(`
+      SELECT id, type, title, subtitle, project, created_at, importance, files_modified
+      FROM observations WHERE ${typeWheres.join(' AND ')}
+      ORDER BY created_at_epoch DESC LIMIT ?
+    `).all(...typeParams);
+    for (const r of typeRows) results.push({ source: 'obs', id: r.id, type: r.type, title: r.title, subtitle: r.subtitle, project: r.project, date: r.created_at, importance: r.importance, files_modified: r.files_modified, score: 0, snippet: '' });
+  }
+
+  // ── Cross-source normalize + sort ──
+  if (isCrossSource && results.length > 0 && ftsQuery) normalizeCrossSourceScores(results, 'source');
+  if (isCrossSource && results.length > 0) {
+    if (ftsQuery) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+    else if (crossSourceEpochSortNoFts) results.sort((a, b) => (b.created_at_epoch ?? 0) - (a.created_at_epoch ?? 0));
+  }
+
+  // ── Context re-rank + superseded marking (markSuperseded is pure stale-tagging) ──
+  const hasObs = results.some((r) => r.source === 'obs');
+  const rerankGate = rerankPolicy === 'mcp' ? ((ftsQuery || isDeep) && hasObs) : hasObs;
+  if (rerankGate) {
+    const obsResults = results.filter((r) => r.source === 'obs');
+    const doReRank = rerankPolicy === 'mcp' ? (ftsQuery && !deepReranked) : !deepReranked;
+    if (doReRank) reRankWithContext(db, obsResults, rerankProject);
+    markSuperseded(obsResults);
+    const doReSort = rerankPolicy === 'mcp' ? (ftsQuery && !deepReranked) : isCrossSource;
+    if (doReSort) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+  }
+
+  // ── Tier post-filter, MCP position: after re-rank, on the merged set ──
+  if (tier && tierPosition === 'late') {
+    const filtered = applyTierFilter(db, results, { tier, sourceKey: 'source', currentProject: tierProject });
+    results.length = 0; results.push(...filtered);
+  }
+
+  // ── User-requested sort (after relevance scoring) ──
+  applyUserSort(results, sort);
+
+  // ── Count + paginate + ~Nt enrich. preFinalizeCount lets the CLI adapter
+  //    distinguish "nothing matched" from "this page is empty" (its two messages). ──
+  const preFinalizeCount = results.length;
+  const { total, page } = finalizeSearchPage(db, results, {
+    isDeep, offset, limit, effectiveSource, ftsQuery, orFallbackFired,
+    project, obsType, importance, branch, epochFrom, epochTo, includeNoise,
+  });
+
+  return {
+    page, total, preFinalizeCount, isDeep, escalated, escalatedObsCount,
+    variants: deepVariants, reranked: deepReranked, orFallbackFired, effectiveSource, ftsQuery,
+  };
+}

package/mem-cli.mjs

@@ -8,8 +8,8 @@ import { truncate, typeIcon, inferProject, scrubSecrets } from './utils.mjs';
 import { resolveProject } from './project-utils.mjs';
 import { TIER_CASE_SQL, tierSqlParams } from './tier.mjs';
 import { _resetVocabCache } from './tfidf.mjs';
-import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './server-internals.mjs';
-import { searchObservationsHybrid, countSearchTotal, attachBodyTokens } from './search-engine.mjs';
+import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './search-scoring.mjs';
+import { searchObservationsHybrid } from './search-engine.mjs';
 import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady } from './deep-search.mjs';
 import { ensureRegistryDb, upsertResource } from './registry.mjs';
 import { searchResources } from './registry-retriever.mjs';
@@ -37,7 +37,7 @@ import { saveObservation } from './lib/save-observation.mjs';
 import { rebuildObservationDerived } from './lib/observation-write.mjs';
 import { recallByFile } from './lib/recall-core.mjs';
 import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentTimeline, fetchTimelineWindow } from './lib/timeline-core.mjs';
-import { buildSearchFtsQuery, parseDateBounds, computePerSourceWindow, effectiveObsFtsQuery, searchSessionsFts, searchPromptsFts, normalizeCrossSourceScores, applyUserSort, applyTierFilter } from './lib/search-core.mjs';
+import { buildSearchFtsQuery, parseDateBounds, coreRunSearchPipeline } from './lib/search-core.mjs';
 import { AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
 import { countRecentHookErrors } from './lib/hook-telemetry.mjs';
 import { computeCitationFunnelTrend } from './lib/citation-tracker.mjs';
@@ -156,116 +156,51 @@ async function cmdSearch(db, args, { llm } = {}) {
     ? 'observations'
     : (source || ((type || tier || minImportance || branch) ? 'observations' : null));
 
-  // Cross-source mode: each source needs more candidates than the final limit
-  // so the post-merge sort has room to pick the best from each (shared sizing
-  // with mem_search — without this, obs gets systematically squeezed out by
-  // sessions). Over-fetch from offset 0; --offset applies ONCE at the final
-  // slice below (see computePerSourceWindow for the #8217/#8638 rationale).
-  const isCrossSourceMode = !effectiveSource;
-  const { perSourceLimit, perSourceOffset } = computePerSourceWindow(limit, offset);
-
-  const results = [];
-  // Tracks whether AND returned 0 and OR recovered non-empty. Mirrors server.mjs
-  // ctx.orFallbackFired so the header can surface a "(relaxed AND→OR)" hint.
-  let orFallbackFired = false;
-
-  let deepVariants = null;
-  let isReranked = false;
-  let isDeep = deepMode === 'deep';
-
-  // Search observations — shared engine with server.mjs (#8198/#8212 paired-path fix)
-  if (!effectiveSource || effectiveSource === 'observations') {
-    const obsCtx = {
-      ftsQuery,
-      args: {
-        project: project || null,
-        obs_type: type || null,
-        importance: minImportance || null,
-        branch: branch || null,
-        include_noise: includeNoise,
-      },
-      epochFrom: dateFrom,
-      epochTo: dateTo,
-      perSourceLimit,
-      perSourceOffset,
-      currentProject: project ? null : inferProject(),
-      limit,
-      orFallbackFired: false,
-    };
-
-    const runDeep = async ({ auto = false } = {}) => {
-      const ds = await deepSearch(db, {
-        query,
-        project: project || null,
-        type: type || null,
-        importance: minImportance || null,
-        branch: branch || null,
-        includeNoise,
-        epochFrom: dateFrom,
-        epochTo: dateTo,
-        limit: perSourceLimit,
-        currentProject: project ? null : inferProject(),
-      }, llm ? { llm, rerank: rerank && !auto } : { auto, rerank: rerank && !auto });
-      deepVariants = ds.variants;
-      isReranked = ds.reranked;
-      if (deepVariants.length > 1) {
-        process.stderr.write(`[mem] Deep search: rewrote into ${deepVariants.length} query variants, RRF-fused\n`);
-      } else {
-        process.stderr.write('[mem] Deep search: rewrite returned no usable variants; used original query only\n');
-      }
-      if (rerank && !auto) {
-        process.stderr.write(ds.reranked
-          ? '[mem] Deep search: LLM-reranked the fused top-20\n'
-          : '[mem] Deep search: rerank produced no usable order; kept fused order\n');
-      }
-      return ds.results;
-    };
-
-    let obsResults;
-    if (deepMode === 'deep') {
-      obsResults = await runDeep();
-    } else {
-      obsResults = searchObservationsHybrid(db, obsCtx);
-      if (obsCtx.orFallbackFired) orFallbackFired = true;
-      if (deepMode === 'auto' && autoDeepLlmReady(process.env, llm) && shouldEscalateToDeep(obsResults, obsCtx, { db, project: project || null })) {
-        process.stderr.write(`[mem] auto-escalated to deep search (weak results: ${obsResults.length} hits)\n`);
-        obsResults = await runDeep({ auto: true });
-        isDeep = true;
-      }
-    }
-    for (const r of obsResults) results.push({ ...r, _source: 'obs', score: r.score ?? 0 });
-
-    // Tier post-filter — applied to ALL obs results from the engine.
-    if (tier) {
-      const filtered = applyTierFilter(db, results, { tier, sourceKey: '_source', currentProject: project || inferProject() });
-      results.length = 0;
-      results.push(...filtered);
-    }
-  }
-
-  // Search sessions (shared engine with MCP mem_search — lib/search-core.mjs)
-  if ((!effectiveSource || effectiveSource === 'sessions') && !isDeep) {
-    try {
-      const sessRows = searchSessionsFts(db, {
-        ftsQuery, project, projectBoost: project ? null : inferProject(),
-        epochFrom: dateFrom, epochTo: dateTo, perSourceLimit, perSourceOffset,
-      });
-      for (const r of sessRows) results.push({ ...r, _source: 'session' });
-    } catch { /* session FTS may not exist in older DBs */ }
-  }
-
-  // Search prompts (shared engine incl. CJK precision gate + LIKE fallback)
-  if ((!effectiveSource || effectiveSource === 'prompts') && !isDeep) {
-    try {
-      const promptRows = searchPromptsFts(db, {
-        query, ftsQuery, project,
-        epochFrom: dateFrom, epochTo: dateTo, perSourceLimit, perSourceOffset,
-      });
-      for (const r of promptRows) results.push({ ...r, _source: 'prompt' });
-    } catch { /* prompt FTS may not exist in older DBs */ }
-  }
-
-  if (results.length === 0) {
+  const res = await coreRunSearchPipeline(
+    {
+      db, currentProject: project ? null : inferProject(), env: process.env,
+      searchObservationsHybrid, deepSearch, shouldEscalateToDeep, autoDeepLlmReady,
+      reRankWithContext, markSuperseded, llm,
+    },
+    {
+      query, ftsQuery, effectiveSource, deepMode, rerank,
+      limit, offset, project: project || null, obsType: type, importance: minImportance,
+      branch, includeNoise, epochFrom: dateFrom, epochTo: dateTo, sort, tier,
+      // ── CLI surface policy ──
+      obsTypeFallback: false,            // #8217 removed list-by-type fallback from the CLI
+      crossSourceEpochSortNoFts: false,  // CLI never reaches cross-source with empty ftsQuery (fails earlier)
+      rerankPolicy: 'cli',               // re-rank/supersede on any obs; re-sort gated on cross-source
+      rerankProject: project || inferProject(),
+      recentListingNoFts: false,
+      tolerateMissingFts: true,          // pre-FTS legacy DBs: swallow session/prompt FTS errors
+      tierPosition: 'early',             // tier filter inside the obs block (before sessions/prompts)
+      tierProject: project || inferProject(),
+    }
+  );
+  const isDeep = res.isDeep;
+  const orFallbackFired = res.orFallbackFired;
+  const deepVariants = res.variants;
+  const paged = res.page;
+  const total = res.total;
+
+  // Deep / escalation observability on stderr — reconstructed from core signals.
+  // The CLI emitted these inline in runDeep; same strings, same order (escalation →
+  // variants → rerank). rerank is only ever true on explicit --deep (never auto).
+  if (res.escalated) process.stderr.write(`[mem] auto-escalated to deep search (weak results: ${res.escalatedObsCount} hits)\n`);
+  if (isDeep && deepVariants) {
+    process.stderr.write(deepVariants.length > 1
+      ? `[mem] Deep search: rewrote into ${deepVariants.length} query variants, RRF-fused\n`
+      : '[mem] Deep search: rewrite returned no usable variants; used original query only\n');
+  }
+  if (rerank) {
+    process.stderr.write(res.reranked
+      ? '[mem] Deep search: LLM-reranked the fused top-20\n'
+      : '[mem] Deep search: rerank produced no usable order; kept fused order\n');
+  }
+
+  // "nothing matched" (no offset) vs "this page is empty" (with offset) — the two
+  // CLI messages. preFinalizeCount is the pre-pagination population (post-tier).
+  if (res.preFinalizeCount === 0) {
     if (jsonOutput) {
       out(JSON.stringify({ query, total: 0, returned: 0, offset, limit, deep: isDeep, variants: isDeep ? deepVariants : undefined, results: [] }));
     } else {
@@ -274,60 +209,6 @@ async function cmdSearch(db, args, { llm } = {}) {
     return;
   }
 
-  // Cross-source score normalization (shared with mem_search).
-  // ftsQuery gate prevents normalization when scores are all 0 (no-FTS path).
-  const isCrossSource = isCrossSourceMode;
-  if (isCrossSource && results.length > 0 && ftsQuery) {
-    normalizeCrossSourceScores(results, '_source');
-    results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
-  }
-
-  // Context re-ranking + superseded marking (aligned with MCP mem_search)
-  const obsResults = results.filter(r => r._source === 'obs');
-  if (obsResults.length > 0) {
-    // reRankWithContext/markSuperseded expect source='obs' — alias _source for compatibility
-    for (const r of obsResults) r.source = 'obs';
-    // Explicit LLM rerank order is final — skip file-context re-rank when reranked
-    // (paired-path with mem_search; markSuperseded still runs for stale-tagging).
-    if (!isReranked) reRankWithContext(db, obsResults, project || inferProject());
-    markSuperseded(obsResults);
-    if (isCrossSource) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
-  }
-
-  // Apply user-requested sort (after relevance scoring; shared with mem_search)
-  applyUserSort(results, sort);
-
-  // Trim to limit with offset. The engine always received perSourceOffset=0 and
-  // over-fetched (see above), so the merged+reranked `results` start at row 0 and
-  // the offset is applied exactly ONCE here — for every mode.
-  //
-  // `total` must be the TRUE population, independent of --limit/--offset (else the
-  // over-fetched candidate count grew with the page and broke the "N of M" /
-  // pagination contract). countSearchTotal mirrors each source's MATCH+filters;
-  // clamp to >= results.length so it never understates the rows actually shown
-  // (vector/concept augmentation can add obs rows beyond the FTS count).
-  // For --deep the population is the fused variant result set: deepSearch already
-  // returned all fused rows (capped at perSourceLimit) and they are the only rows
-  // in `results` (deep is obs-only). countSearchTotal would instead count the
-  // ORIGINAL query's FTS matches — wrong, and ~0 on the vocabulary-mismatch
-  // queries deep exists for, which falsely shrinks the "N of M" total (F1).
-  const total = isDeep
-    ? results.length
-    : Math.max(countSearchTotal(db, {
-      effectiveSource,
-      ftsQuery,
-      obsFtsQuery: effectiveObsFtsQuery(ftsQuery, orFallbackFired),
-      args: { project: project || null, obs_type: type || null, importance: minImportance || null, branch: branch || null },
-      project: project || null,
-      epochFrom: dateFrom,
-      epochTo: dateTo,
-      includeNoise,
-    }), results.length);
-  const paged = results.slice(offset, offset + limit);
-  // Enrich the final page with the ~Nt fetch-cost hint (paired with MCP mem_search; #8654 both
-  // source keys handled). Batch-fetches heavy obs fields by id — no-op on an empty page.
-  attachBodyTokens(db, paged);
-
   if (paged.length === 0) {
     if (jsonOutput) {
       out(JSON.stringify({ query, total, returned: 0, offset, limit, deep: isDeep, variants: isDeep ? deepVariants : undefined, results: [] }));
@@ -337,24 +218,24 @@ async function cmdSearch(db, args, { llm } = {}) {
     return;
   }
 
-  // paired-path with server.mjs formatSearchOutput (#8198): "N of M" total when paged < total.
+  // "N of M" total when paged < total (paired-path with server.mjs formatSearchOutput, #8198).
   const showTime = sort === 'time';
-  const hasMixed = paged.some(r => r._source === 'session' || r._source === 'prompt');
+  const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt');
   // Suppressed when --or was explicit — user already asked for OR, no "fallback" there.
   const fallbackHint = orFallbackFired && !useOr ? ' (relaxed AND→OR)' : '';
 
   if (jsonOutput) {
     const items = paged.map(r => {
       const base = {
-        source: r._source,
+        source: r.source,
         id: r.id,
         created_at: r.created_at,
         score: r.score ?? null,
       };
-      if (r._source === 'session') {
+      if (r.source === 'session') {
         return { ...base, request: r.request || null, completed: r.completed || null, project: r.project || null };
       }
-      if (r._source === 'prompt') {
+      if (r.source === 'prompt') {
         return { ...base, prompt_text: r.prompt_text || null };
       }
       return {
@@ -392,10 +273,10 @@ async function cmdSearch(db, args, { llm } = {}) {
   const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
   for (const r of paged) {
     const timeStr = showTime && r.created_at_epoch ? ` (${relativeTime(r.created_at_epoch)})` : '';
-    if (r._source === 'session') {
+    if (r.source === 'session') {
       const date = fmtDateShort(r.created_at);
       out(`S#${r.id} 📋 ${date}${timeStr} ${truncate(r.request || r.completed || '(no summary)', 80)}${tok(r)}`);
-    } else if (r._source === 'prompt') {
+    } else if (r.source === 'prompt') {
       const date = fmtDateShort(r.created_at);
       out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}${tok(r)}`);
     } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "3.7.0",
+  "version": "3.7.1",
   "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
   "type": "module",
   "packageManager": "npm@10.9.2",
@@ -28,7 +28,7 @@
     "cli-path.mjs",
     "mem-cli.mjs",
     "server.mjs",
-    "server-internals.mjs",
+    "search-scoring.mjs",
     "search-engine.mjs",
     "deep-search.mjs",
     "rerank.mjs",