claude-mem-lite 3.6.0 → 3.7.1

package/mem-cli.mjs

@@ -8,9 +8,9 @@ import { truncate, typeIcon, inferProject, scrubSecrets } from './utils.mjs';
 import { resolveProject } from './project-utils.mjs';
 import { TIER_CASE_SQL, tierSqlParams } from './tier.mjs';
 import { _resetVocabCache } from './tfidf.mjs';
-import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './server-internals.mjs';
-import { searchObservationsHybrid, countSearchTotal, attachBodyTokens } from './search-engine.mjs';
-import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady, hasEscalatableCorpus } from './deep-search.mjs';
+import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './search-scoring.mjs';
+import { searchObservationsHybrid } from './search-engine.mjs';
+import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady } from './deep-search.mjs';
 import { ensureRegistryDb, upsertResource } from './registry.mjs';
 import { searchResources } from './registry-retriever.mjs';
 import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
@@ -37,7 +37,7 @@ import { saveObservation } from './lib/save-observation.mjs';
 import { rebuildObservationDerived } from './lib/observation-write.mjs';
 import { recallByFile } from './lib/recall-core.mjs';
 import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentTimeline, fetchTimelineWindow } from './lib/timeline-core.mjs';
-import { buildSearchFtsQuery, parseDateBounds, computePerSourceWindow, effectiveObsFtsQuery, searchSessionsFts, searchPromptsFts, normalizeCrossSourceScores, applyUserSort, applyTierFilter } from './lib/search-core.mjs';
+import { buildSearchFtsQuery, parseDateBounds, coreRunSearchPipeline } from './lib/search-core.mjs';
 import { AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
 import { countRecentHookErrors } from './lib/hook-telemetry.mjs';
 import { computeCitationFunnelTrend } from './lib/citation-tracker.mjs';
@@ -156,116 +156,51 @@ async function cmdSearch(db, args, { llm } = {}) {
     ? 'observations'
     : (source || ((type || tier || minImportance || branch) ? 'observations' : null));
 
-  // Cross-source mode: each source needs more candidates than the final limit
-  // so the post-merge sort has room to pick the best from each (shared sizing
-  // with mem_search — without this, obs gets systematically squeezed out by
-  // sessions). Over-fetch from offset 0; --offset applies ONCE at the final
-  // slice below (see computePerSourceWindow for the #8217/#8638 rationale).
-  const isCrossSourceMode = !effectiveSource;
-  const { perSourceLimit, perSourceOffset } = computePerSourceWindow(limit, offset);
-
-  const results = [];
-  // Tracks whether AND returned 0 and OR recovered non-empty. Mirrors server.mjs
-  // ctx.orFallbackFired so the header can surface a "(relaxed AND→OR)" hint.
-  let orFallbackFired = false;
-
-  let deepVariants = null;
-  let isReranked = false;
-  let isDeep = deepMode === 'deep';
-
-  // Search observations — shared engine with server.mjs (#8198/#8212 paired-path fix)
-  if (!effectiveSource || effectiveSource === 'observations') {
-    const obsCtx = {
-      ftsQuery,
-      args: {
-        project: project || null,
-        obs_type: type || null,
-        importance: minImportance || null,
-        branch: branch || null,
-        include_noise: includeNoise,
-      },
-      epochFrom: dateFrom,
-      epochTo: dateTo,
-      perSourceLimit,
-      perSourceOffset,
-      currentProject: project ? null : inferProject(),
-      limit,
-      orFallbackFired: false,
-    };
-
-    const runDeep = async ({ auto = false } = {}) => {
-      const ds = await deepSearch(db, {
-        query,
-        project: project || null,
-        type: type || null,
-        importance: minImportance || null,
-        branch: branch || null,
-        includeNoise,
-        epochFrom: dateFrom,
-        epochTo: dateTo,
-        limit: perSourceLimit,
-        currentProject: project ? null : inferProject(),
-      }, llm ? { llm, rerank: rerank && !auto } : { auto, rerank: rerank && !auto });
-      deepVariants = ds.variants;
-      isReranked = ds.reranked;
-      if (deepVariants.length > 1) {
-        process.stderr.write(`[mem] Deep search: rewrote into ${deepVariants.length} query variants, RRF-fused\n`);
-      } else {
-        process.stderr.write('[mem] Deep search: rewrite returned no usable variants; used original query only\n');
-      }
-      if (rerank && !auto) {
-        process.stderr.write(ds.reranked
-          ? '[mem] Deep search: LLM-reranked the fused top-20\n'
-          : '[mem] Deep search: rerank produced no usable order; kept fused order\n');
-      }
-      return ds.results;
-    };
-
-    let obsResults;
-    if (deepMode === 'deep') {
-      obsResults = await runDeep();
-    } else {
-      obsResults = searchObservationsHybrid(db, obsCtx);
-      if (obsCtx.orFallbackFired) orFallbackFired = true;
-      if (deepMode === 'auto' && autoDeepLlmReady(process.env, llm) && shouldEscalateToDeep(obsResults, obsCtx) && hasEscalatableCorpus(db, project || null)) {
-        process.stderr.write(`[mem] auto-escalated to deep search (weak results: ${obsResults.length} hits)\n`);
-        obsResults = await runDeep({ auto: true });
-        isDeep = true;
-      }
-    }
-    for (const r of obsResults) results.push({ ...r, _source: 'obs', score: r.score ?? 0 });
-
-    // Tier post-filter — applied to ALL obs results from the engine.
-    if (tier) {
-      const filtered = applyTierFilter(db, results, { tier, sourceKey: '_source', currentProject: project || inferProject() });
-      results.length = 0;
-      results.push(...filtered);
-    }
-  }
-
-  // Search sessions (shared engine with MCP mem_search — lib/search-core.mjs)
-  if ((!effectiveSource || effectiveSource === 'sessions') && !isDeep) {
-    try {
-      const sessRows = searchSessionsFts(db, {
-        ftsQuery, project, projectBoost: project ? null : inferProject(),
-        epochFrom: dateFrom, epochTo: dateTo, perSourceLimit, perSourceOffset,
-      });
-      for (const r of sessRows) results.push({ ...r, _source: 'session' });
-    } catch { /* session FTS may not exist in older DBs */ }
-  }
-
-  // Search prompts (shared engine incl. CJK precision gate + LIKE fallback)
-  if ((!effectiveSource || effectiveSource === 'prompts') && !isDeep) {
-    try {
-      const promptRows = searchPromptsFts(db, {
-        query, ftsQuery, project,
-        epochFrom: dateFrom, epochTo: dateTo, perSourceLimit, perSourceOffset,
-      });
-      for (const r of promptRows) results.push({ ...r, _source: 'prompt' });
-    } catch { /* prompt FTS may not exist in older DBs */ }
-  }
-
-  if (results.length === 0) {
+  const res = await coreRunSearchPipeline(
+    {
+      db, currentProject: project ? null : inferProject(), env: process.env,
+      searchObservationsHybrid, deepSearch, shouldEscalateToDeep, autoDeepLlmReady,
+      reRankWithContext, markSuperseded, llm,
+    },
+    {
+      query, ftsQuery, effectiveSource, deepMode, rerank,
+      limit, offset, project: project || null, obsType: type, importance: minImportance,
+      branch, includeNoise, epochFrom: dateFrom, epochTo: dateTo, sort, tier,
+      // ── CLI surface policy ──
+      obsTypeFallback: false,            // #8217 removed list-by-type fallback from the CLI
+      crossSourceEpochSortNoFts: false,  // CLI never reaches cross-source with empty ftsQuery (fails earlier)
+      rerankPolicy: 'cli',               // re-rank/supersede on any obs; re-sort gated on cross-source
+      rerankProject: project || inferProject(),
+      recentListingNoFts: false,
+      tolerateMissingFts: true,          // pre-FTS legacy DBs: swallow session/prompt FTS errors
+      tierPosition: 'early',             // tier filter inside the obs block (before sessions/prompts)
+      tierProject: project || inferProject(),
+    }
+  );
+  const isDeep = res.isDeep;
+  const orFallbackFired = res.orFallbackFired;
+  const deepVariants = res.variants;
+  const paged = res.page;
+  const total = res.total;
+
+  // Deep / escalation observability on stderr — reconstructed from core signals.
+  // The CLI emitted these inline in runDeep; same strings, same order (escalation →
+  // variants → rerank). rerank is only ever true on explicit --deep (never auto).
+  if (res.escalated) process.stderr.write(`[mem] auto-escalated to deep search (weak results: ${res.escalatedObsCount} hits)\n`);
+  if (isDeep && deepVariants) {
+    process.stderr.write(deepVariants.length > 1
+      ? `[mem] Deep search: rewrote into ${deepVariants.length} query variants, RRF-fused\n`
+      : '[mem] Deep search: rewrite returned no usable variants; used original query only\n');
+  }
+  if (rerank) {
+    process.stderr.write(res.reranked
+      ? '[mem] Deep search: LLM-reranked the fused top-20\n'
+      : '[mem] Deep search: rerank produced no usable order; kept fused order\n');
+  }
+
+  // "nothing matched" (no offset) vs "this page is empty" (with offset) — the two
+  // CLI messages. preFinalizeCount is the pre-pagination population (post-tier).
+  if (res.preFinalizeCount === 0) {
     if (jsonOutput) {
       out(JSON.stringify({ query, total: 0, returned: 0, offset, limit, deep: isDeep, variants: isDeep ? deepVariants : undefined, results: [] }));
     } else {
@@ -274,60 +209,6 @@ async function cmdSearch(db, args, { llm } = {}) {
     return;
   }
 
-  // Cross-source score normalization (shared with mem_search).
-  // ftsQuery gate prevents normalization when scores are all 0 (no-FTS path).
-  const isCrossSource = isCrossSourceMode;
-  if (isCrossSource && results.length > 0 && ftsQuery) {
-    normalizeCrossSourceScores(results, '_source');
-    results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
-  }
-
-  // Context re-ranking + superseded marking (aligned with MCP mem_search)
-  const obsResults = results.filter(r => r._source === 'obs');
-  if (obsResults.length > 0) {
-    // reRankWithContext/markSuperseded expect source='obs' — alias _source for compatibility
-    for (const r of obsResults) r.source = 'obs';
-    // Explicit LLM rerank order is final — skip file-context re-rank when reranked
-    // (paired-path with mem_search; markSuperseded still runs for stale-tagging).
-    if (!isReranked) reRankWithContext(db, obsResults, project || inferProject());
-    markSuperseded(obsResults);
-    if (isCrossSource) results.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
-  }
-
-  // Apply user-requested sort (after relevance scoring; shared with mem_search)
-  applyUserSort(results, sort);
-
-  // Trim to limit with offset. The engine always received perSourceOffset=0 and
-  // over-fetched (see above), so the merged+reranked `results` start at row 0 and
-  // the offset is applied exactly ONCE here — for every mode.
-  //
-  // `total` must be the TRUE population, independent of --limit/--offset (else the
-  // over-fetched candidate count grew with the page and broke the "N of M" /
-  // pagination contract). countSearchTotal mirrors each source's MATCH+filters;
-  // clamp to >= results.length so it never understates the rows actually shown
-  // (vector/concept augmentation can add obs rows beyond the FTS count).
-  // For --deep the population is the fused variant result set: deepSearch already
-  // returned all fused rows (capped at perSourceLimit) and they are the only rows
-  // in `results` (deep is obs-only). countSearchTotal would instead count the
-  // ORIGINAL query's FTS matches — wrong, and ~0 on the vocabulary-mismatch
-  // queries deep exists for, which falsely shrinks the "N of M" total (F1).
-  const total = isDeep
-    ? results.length
-    : Math.max(countSearchTotal(db, {
-      effectiveSource,
-      ftsQuery,
-      obsFtsQuery: effectiveObsFtsQuery(ftsQuery, orFallbackFired),
-      args: { project: project || null, obs_type: type || null, importance: minImportance || null, branch: branch || null },
-      project: project || null,
-      epochFrom: dateFrom,
-      epochTo: dateTo,
-      includeNoise,
-    }), results.length);
-  const paged = results.slice(offset, offset + limit);
-  // Enrich the final page with the ~Nt fetch-cost hint (paired with MCP mem_search; #8654 both
-  // source keys handled). Batch-fetches heavy obs fields by id — no-op on an empty page.
-  attachBodyTokens(db, paged);
-
   if (paged.length === 0) {
     if (jsonOutput) {
       out(JSON.stringify({ query, total, returned: 0, offset, limit, deep: isDeep, variants: isDeep ? deepVariants : undefined, results: [] }));
@@ -337,24 +218,24 @@ async function cmdSearch(db, args, { llm } = {}) {
     return;
   }
 
-  // paired-path with server.mjs formatSearchOutput (#8198): "N of M" total when paged < total.
+  // "N of M" total when paged < total (paired-path with server.mjs formatSearchOutput, #8198).
   const showTime = sort === 'time';
-  const hasMixed = paged.some(r => r._source === 'session' || r._source === 'prompt');
+  const hasMixed = paged.some(r => r.source === 'session' || r.source === 'prompt');
   // Suppressed when --or was explicit — user already asked for OR, no "fallback" there.
   const fallbackHint = orFallbackFired && !useOr ? ' (relaxed AND→OR)' : '';
 
   if (jsonOutput) {
     const items = paged.map(r => {
       const base = {
-        source: r._source,
+        source: r.source,
         id: r.id,
         created_at: r.created_at,
         score: r.score ?? null,
       };
-      if (r._source === 'session') {
+      if (r.source === 'session') {
         return { ...base, request: r.request || null, completed: r.completed || null, project: r.project || null };
       }
-      if (r._source === 'prompt') {
+      if (r.source === 'prompt') {
         return { ...base, prompt_text: r.prompt_text || null };
       }
       return {
@@ -392,10 +273,10 @@ async function cmdSearch(db, args, { llm } = {}) {
   const tok = r => (r.bodyTokens ? ` ~${r.bodyTokens}t` : '');
   for (const r of paged) {
     const timeStr = showTime && r.created_at_epoch ? ` (${relativeTime(r.created_at_epoch)})` : '';
-    if (r._source === 'session') {
+    if (r.source === 'session') {
       const date = fmtDateShort(r.created_at);
       out(`S#${r.id} 📋 ${date}${timeStr} ${truncate(r.request || r.completed || '(no summary)', 80)}${tok(r)}`);
-    } else if (r._source === 'prompt') {
+    } else if (r.source === 'prompt') {
       const date = fmtDateShort(r.created_at);
       out(`P#${r.id} 💬 ${date}${timeStr} ${truncate(r.prompt_text || '(empty)', 80)}${tok(r)}`);
     } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "3.6.0",
+  "version": "3.7.1",
   "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
   "type": "module",
   "packageManager": "npm@10.9.2",
@@ -28,7 +28,7 @@
     "cli-path.mjs",
     "mem-cli.mjs",
     "server.mjs",
-    "server-internals.mjs",
+    "search-scoring.mjs",
     "search-engine.mjs",
     "deep-search.mjs",
     "rerank.mjs",
@@ -69,7 +69,10 @@
     "lib/file-intel.mjs",
     "lib/reread-guard.mjs",
     "lib/metrics.mjs",
+    "lib/lesson-idents.mjs",
     "lib/binding-probe.mjs",
+    "lib/proc-lock.mjs",
+    "lib/atomic-write.mjs",
     "lib/mem-override.mjs",
     "lib/save-observation.mjs",
     "lib/observation-write.mjs",
@@ -131,6 +134,7 @@
     "scripts/post-tool-use.sh",
     "scripts/user-prompt-search.js",
     "scripts/pre-tool-recall.js",
+    "scripts/post-tool-recall.js",
     "scripts/pre-skill-bridge.js",
     "scripts/prompt-search-utils.mjs",
     "scripts/hook-launcher.mjs",

package/schema.mjs

@@ -6,7 +6,7 @@ import Database from 'better-sqlite3';
 import { homedir } from 'os';
 import { join } from 'path';
 import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, chmodSync } from 'fs';
-import { OBS_FTS_COLUMNS } from './utils.mjs';
+import { OBS_FTS_COLUMNS, debugCatch } from './utils.mjs';
 
 // DATA location — DB, managed resources, registry DB, runtime/. Honors
 // CLAUDE_MEM_DIR so users can relocate state to a larger/faster volume.
@@ -79,7 +79,16 @@ export const CODE_DIR = join(homedir(), '.claude-mem-lite');
 // New TABLE (not a column) reached via CORE_SCHEMA's CREATE TABLE IF NOT EXISTS on the
 // forced migration pass; LATEST_MIGRATION_COLUMN unchanged (no new column) — same
 // pattern as v35/v36.
-export const CURRENT_SCHEMA_VERSION = 38;
+// v39 (audit P1-5): migration_cleanups table — a sentinel registry that makes the
+// one-shot DATA cleanups (orphan deletes, project-name normalization) RETRYABLE.
+// They previously ran inside the version-gated migration body and were swallowed
+// on failure AFTER the version stamp committed, so a failed cleanup could never
+// re-run (the fast-path then skipped the whole body). They now run via
+// runDeferredCleanups() on every ensureDb, each gated by a done-marker row: a
+// failure leaves the marker unset and retries on the next open. New TABLE via
+// CORE_SCHEMA on the forced pass; LATEST_MIGRATION_COLUMN unchanged (no new
+// column) — same pattern as v35/v36/v38.
+export const CURRENT_SCHEMA_VERSION = 39;
 
 // Sentinel column for the LATEST migration set. The fast-path uses this to
 // self-heal half-migrated DBs — schema_version bumped but column ALTERs rolled
@@ -185,6 +194,11 @@ const CORE_SCHEMA = `
     cited_n INTEGER NOT NULL DEFAULT 0,
     PRIMARY KEY (project, memory_session_id)
   );
+
+  CREATE TABLE IF NOT EXISTS migration_cleanups (
+    name TEXT PRIMARY KEY,
+    done_at_epoch INTEGER NOT NULL
+  );
 `;
 
 // Column migrations (idempotent — only swallow "duplicate column" errors)
@@ -556,18 +570,8 @@ export function initSchema(db) {
     }
   } catch { /* non-critical — migration can retry on next open */ }
 
-  // v35 (v2.87.0) P1: one-shot cleanup of orphaned observation_files. Same root
-  // cause as the v28 observation_vectors cleanup below — until the warm-start
-  // fast-path FK fix (initSchema early returns now restore foreign_keys=ON),
-  // ensureDb() handles ran with FK OFF, so ON DELETE CASCADE never fired and
-  // junction rows leaked (live DB: 6440/9569 = 67% orphans). Idempotent (NOT IN
-  // is empty on a clean DB); runs once per version bump via the fast-path gate.
-  try {
-    db.prepare(`
-      DELETE FROM observation_files
-      WHERE obs_id NOT IN (SELECT id FROM observations)
-    `).run();
-  } catch { /* non-critical — table-missing path handled by earlier CREATE */ }
+  // observation_files orphan cleanup moved to runDeferredCleanups() (audit P1-5):
+  // it now runs retryably outside the version fast-path. See DEFERRED_CLEANUPS.
 
   // Observation vectors table for TF-IDF vector search
   db.exec(`
@@ -582,16 +586,7 @@ export function initSchema(db) {
 
   db.exec(`CREATE INDEX IF NOT EXISTS idx_obs_vectors_version ON observation_vectors(vocab_version)`);
 
-  // v28 (v2.47) P0-1: one-shot cleanup of orphaned observation_vectors.
-  // Live DBs accumulated 44% orphans even with ON DELETE CASCADE because
-  // early migrations ran with `foreign_keys=OFF` and deletes skipped cascade.
-  // Idempotent (NOT IN is empty on a clean DB), runs once per ensureDb().
-  try {
-    db.prepare(`
-      DELETE FROM observation_vectors
-      WHERE observation_id NOT IN (SELECT id FROM observations)
-    `).run();
-  } catch { /* non-critical — table-missing path handled by earlier CREATE */ }
+  // observation_vectors orphan cleanup moved to runDeferredCleanups() (audit P1-5).
 
   // Persisted vocabulary for stable TF-IDF vector indexing
   db.exec(`
@@ -605,46 +600,9 @@ export function initSchema(db) {
   `);
   db.exec('CREATE INDEX IF NOT EXISTS idx_vocab_state_version ON vocab_state(version)');
 
-  // Project name normalization: migrate short names ("mem") to canonical form ("projects--mem")
-  // Strategy: exact suffix match first, then substring match for package-name aliases
-  // Idempotent: only runs when short-name records exist
-  try {
-    const shortProjects = db.prepare(`
-      SELECT DISTINCT project FROM observations
-      WHERE project NOT LIKE '%--_%' AND project != '' AND project IS NOT NULL
-      UNION
-      SELECT DISTINCT project FROM sdk_sessions
-      WHERE project NOT LIKE '%--_%' AND project != '' AND project IS NOT NULL
-    `).all();
-    if (shortProjects.length > 0) {
-      const normalize = db.transaction(() => {
-        for (const { project: shortName } of shortProjects) {
-          // Strategy 1: exact suffix match (e.g., "mem" → "projects--mem")
-          let canonical = db.prepare(
-            `SELECT project FROM observations WHERE project LIKE ? GROUP BY project ORDER BY COUNT(*) DESC LIMIT 1`
-          ).get(`%--${shortName}`);
-          // Strategy 2: substring match for aliases (e.g., "claude-mem-lite" → match project containing "mem")
-          // Extract the most distinctive token from the short name for fuzzy matching
-          if (!canonical) {
-            const tokens = shortName.split(/[-_.]/).filter(t => t.length >= 5);
-            for (const token of tokens) {
-              canonical = db.prepare(
-                `SELECT project FROM observations WHERE project LIKE ? AND project LIKE '%--_%'
-                 GROUP BY project ORDER BY COUNT(*) DESC LIMIT 1`
-              ).get(`%${token}%`);
-              if (canonical) break;
-            }
-          }
-          if (canonical) {
-            for (const table of ['observations', 'sdk_sessions', 'session_summaries']) {
-              db.prepare(`UPDATE ${table} SET project = ? WHERE project = ?`).run(canonical.project, shortName);
-            }
-          }
-        }
-      });
-      normalize();
-    }
-  } catch { /* non-critical — normalization can retry on next open */ }
+  // Project-name normalization moved to runDeferredCleanups() (audit P1-5) — it
+  // now retries on a later open if it fails, instead of being lost behind the
+  // version fast-path. See DEFERRED_CLEANUPS.
 
   // ─── v29 (v2.57.x): session-id mix invariant + lesson-retry stats ─────────
   //
@@ -804,6 +762,96 @@ export function auditSessionConsistency(db, { graceMinutes = 5 } = {}) {
   };
 }
 
+// ─── Deferred one-shot cleanups (retryable) ─────────────────────────────────
+// Idempotent DATA cleanups that must survive a transient failure. They run on
+// EVERY ensureDb (after initSchema), each gated by a row in migration_cleanups:
+// a cleanup that throws leaves its marker unset and retries on the next open —
+// unlike the old version-gated body, where a swallowed failure committed
+// alongside the version stamp and the fast-path then skipped it forever (P1-5).
+const DEFERRED_CLEANUPS = [
+  {
+    // v35 (v2.87.0): orphaned observation_files. ON DELETE CASCADE didn't fire
+    // while early warm-start handles ran with foreign_keys OFF, so junction rows
+    // leaked. Idempotent (NOT IN is empty on a clean DB).
+    name: 'orphan-observation-files',
+    run: (db) => db.prepare(
+      `DELETE FROM observation_files WHERE obs_id NOT IN (SELECT id FROM observations)`
+    ).run(),
+  },
+  {
+    // v28 (v2.47) P0-1: orphaned observation_vectors — same FK-OFF root cause.
+    name: 'orphan-observation-vectors',
+    run: (db) => db.prepare(
+      `DELETE FROM observation_vectors WHERE observation_id NOT IN (SELECT id FROM observations)`
+    ).run(),
+  },
+  {
+    // Project-name normalization: migrate short names ("mem") to canonical
+    // ("projects--mem"). Exact suffix match first, then distinctive-token
+    // substring. Idempotent: only acts on remaining short-name records.
+    name: 'normalize-project-names',
+    run: (db) => {
+      const shortProjects = db.prepare(`
+        SELECT DISTINCT project FROM observations
+        WHERE project NOT LIKE '%--_%' AND project != '' AND project IS NOT NULL
+        UNION
+        SELECT DISTINCT project FROM sdk_sessions
+        WHERE project NOT LIKE '%--_%' AND project != '' AND project IS NOT NULL
+      `).all();
+      if (shortProjects.length === 0) return;
+      const normalize = db.transaction(() => {
+        for (const { project: shortName } of shortProjects) {
+          let canonical = db.prepare(
+            `SELECT project FROM observations WHERE project LIKE ? GROUP BY project ORDER BY COUNT(*) DESC LIMIT 1`
+          ).get(`%--${shortName}`);
+          if (!canonical) {
+            const tokens = shortName.split(/[-_.]/).filter(t => t.length >= 5);
+            for (const token of tokens) {
+              canonical = db.prepare(
+                `SELECT project FROM observations WHERE project LIKE ? AND project LIKE '%--_%'
+                 GROUP BY project ORDER BY COUNT(*) DESC LIMIT 1`
+              ).get(`%${token}%`);
+              if (canonical) break;
+            }
+          }
+          if (canonical) {
+            for (const table of ['observations', 'sdk_sessions', 'session_summaries']) {
+              db.prepare(`UPDATE ${table} SET project = ? WHERE project = ?`).run(canonical.project, shortName);
+            }
+          }
+        }
+      });
+      normalize();
+    },
+  },
+];
+
+/**
+ * Run registered one-shot data cleanups that haven't completed yet. Each is
+ * gated by a row in migration_cleanups, so a transient failure retries on the
+ * next open instead of being silently lost behind the schema-version fast-path
+ * (audit P1-5). Best-effort: never throws — callers open the DB regardless.
+ */
+export function runDeferredCleanups(db) {
+  let done;
+  try {
+    done = new Set(db.prepare('SELECT name FROM migration_cleanups').all().map(r => r.name));
+  } catch {
+    return; // table not present yet (pre-migration open) — nothing to do
+  }
+  const mark = db.prepare('INSERT OR IGNORE INTO migration_cleanups (name, done_at_epoch) VALUES (?, ?)');
+  for (const { name, run } of DEFERRED_CLEANUPS) {
+    if (done.has(name)) continue;
+    try {
+      run(db);
+      mark.run(name, Date.now());
+    } catch (e) {
+      // Leave the marker unset → retried next open. Surface for observability.
+      debugCatch(e, `deferred-cleanup:${name}`);
+    }
+  }
+}
+
 /**
  * Ensure DB directory, database file, and all tables exist.
  * Safe to call from any process (hook or server). Idempotent.
@@ -846,7 +894,13 @@ export function ensureDb() {
   db.pragma('foreign_keys = OFF'); // Enabled after dedup migration
 
   try {
-    return initSchema(db);
+    const ready = initSchema(db);
+    // P1-5: sentinel-gated data cleanups must run on EVERY open (schema.mjs:766).
+    // They were extracted out of initSchema into runDeferredCleanups but never
+    // wired into a production opener — without this call they ran nowhere but
+    // tests, silently halting orphan/normalize hygiene. Best-effort: never throws.
+    runDeferredCleanups(ready);
+    return ready;
   } catch (e) {
     try { db.close(); } catch {}
     throw e;

package/scoring-sql.mjs

@@ -3,6 +3,31 @@
 
 import { buildNotLowSignalSql } from './lib/low-signal-patterns.mjs';
 
+// ─── Why these multipliers exist (read before "simplifying" them) ────────────
+//
+// The recency-decay, type-quality, project-boost, importance, cite, and noise
+// multipliers below encode PRODUCT PRIORS: recent / same-project / important /
+// high-signal-type / frequently-cited memories are more relevant to the CURRENT
+// dev session. A periodic audit tends to flag them as "0-lift dead weight" —
+// resist that on benchmark evidence alone. Measured (audit ②, obs #8773):
+//   * benchmark.mjs --matrix (micro-fixture, now models the full FULL_SCORE
+//     chain): type-quality is the TOP contributor (drop-type ΔnDCG=0.0082,
+//     ΔMRR=0.0166), decay +0.0043 nDCG, importance +0.0012; the chain lifts
+//     hybrid over bm25_only by +0.0093 nDCG / +0.0166 MRR (net 0 queries hurt).
+//     project, access and lesson read exactly 0 — but that is STRUCTURAL: the
+//     fixture is single-project, access_count=0, and has 0 lesson_learned rows,
+//     so it cannot vary those three axes.
+//   * longmemeval.mjs --temporal (n=500, real dates): bit-identical to uniform —
+//     LongMemEval-S windows (mean 27.9d, 74% <30d) are far shorter than these
+//     half-lives, so decay moves no rank there either.
+// Where a multiplier reads 0 it is a benchmark-MISMATCH artifact (the instrument
+// can't vary that axis), NOT proven dead weight. Decision: KEEP them; do NOT
+// delete on "0 lift". Guardrail: the ci-gate `hybrid_over_bm25 >= -0.05` floor
+// (benchmark/ci-gate.mjs) covers the full modelled chain
+// (decay/type/project/importance/access/lesson); cite + noise live on the
+// injection path (hook-memory.mjs) with no recall-benchmark coverage. Genuine
+// validation of the prior-encoding axes needs a labeled real-dev-memory eval.
+
 // ─── Type-Differentiated Recency Decay ──────────────────────────────────────
 
 /** Recency half-life per observation type (in milliseconds) */