claude-mem-lite 2.87.0 → 2.88.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.87.0",
+      "version": "2.88.0",
       "source": "./",
       "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. Alternative to claude-mem with 600x lower cost."
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.87.0",
+  "version": "2.88.0",
   "description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. Alternative to claude-mem with 600x lower cost.",
   "author": {
     "name": "sdsrss"

package/hook.mjs

@@ -26,7 +26,7 @@ import {
   truncate, inferProject, detectBashSignificance,
   extractErrorKeywords, extractFilePaths, isRelatedToEpisode,
   makeEntryDesc, scrubSecrets, stripPrivate, EDIT_TOOLS, debugCatch, debugLog,
-  COMPRESSED_AUTO, COMPRESSED_PENDING_PURGE, isoWeekKey, OBS_BM25,
+  COMPRESSED_AUTO, COMPRESSED_PENDING_PURGE, OBS_BM25,
   computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity,
 } from './utils.mjs';
 import {
@@ -45,6 +45,8 @@ import {
 } from './hook-shared.mjs';
 import { handleLLMEpisode, handleLLMSummary, saveObservation, buildImmediateObservation } from './hook-llm.mjs';
 import { scrubRecord } from './lib/scrub-record.mjs';
+import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
+import { cleanupBroken, decayAndMarkIdle, boostAccessed } from './lib/maintain-core.mjs';
 import {
   extractCitationsFromTranscript,
   extractAllInjected,
@@ -819,65 +821,19 @@ async function handleSessionStart() {
         `).run(Date.now() - 37 * 86400000);
         if (purged.changes > 0) debugLog('DEBUG', 'auto-maintain', `purged ${purged.changes} stale observations`);
 
-        // Cleanup: remove broken observations (no title AND no narrative)
-        const cleaned = db.prepare(`
-          DELETE FROM observations WHERE id IN (
-            SELECT id FROM observations
-            WHERE COALESCE(compressed_into, 0) = 0
-              AND (title IS NULL OR title = '') AND (narrative IS NULL OR narrative = '')
-            LIMIT ${OP_CAP}
-          )
-        `).run();
-        if (cleaned.changes > 0) debugLog('DEBUG', 'auto-maintain', `cleaned ${cleaned.changes} broken observations`);
-
-        // Decay: reduce importance of old, never-accessed observations
-        // v2.56.0 #4: injection_count is a separate engagement signal —
-        // hook-memory.mjs bumps it when the obs is auto-injected into Claude's
-        // context. Pre-v2.56 only checked access_count, so an obs auto-injected
-        // 8x (proven contextually relevant) still got decayed/marked. Adding
-        // `injection_count = 0` treats injection as first-class engagement.
-        const decayed = db.prepare(`
-          UPDATE observations SET importance = MAX(1, COALESCE(importance, 1) - 1)
-          WHERE id IN (
-            SELECT id FROM observations
-            WHERE COALESCE(compressed_into, 0) = 0
-              AND COALESCE(importance, 1) > 1
-              AND COALESCE(access_count, 0) = 0
-              AND COALESCE(injection_count, 0) = 0
-              AND created_at_epoch < ?
-            LIMIT ${OP_CAP}
-          )
-        `).run(STALE_AGE);
-        if (decayed.changes > 0) debugLog('DEBUG', 'auto-maintain', `decayed ${decayed.changes} stale observations`);
-
-        // Mark idle: importance=1, never-accessed, never-injected, old → pending-purge
-        // (will be purged next cycle). v2.56.0 #4: injection_count protects.
-        const idleMarked = db.prepare(`
-          UPDATE observations SET compressed_into = ${COMPRESSED_PENDING_PURGE}
-          WHERE id IN (
-            SELECT id FROM observations
-            WHERE COALESCE(compressed_into, 0) = 0
-              AND COALESCE(importance, 1) = 1
-              AND COALESCE(access_count, 0) = 0
-              AND COALESCE(injection_count, 0) = 0
-              AND created_at_epoch < ?
-            LIMIT ${OP_CAP}
-          )
-        `).run(STALE_AGE);
-        if (idleMarked.changes > 0) debugLog('DEBUG', 'auto-maintain', `marked ${idleMarked.changes} idle as pending-purge`);
-
-        // Boost: increase importance of frequently-accessed observations
-        const boosted = db.prepare(`
-          UPDATE observations SET importance = MIN(3, COALESCE(importance, 1) + 1)
-          WHERE id IN (
-            SELECT id FROM observations
-            WHERE COALESCE(compressed_into, 0) = 0
-              AND COALESCE(access_count, 0) > 3
-              AND COALESCE(importance, 1) < 3
-            LIMIT ${OP_CAP}
-          )
-        `).run();
-        if (boosted.changes > 0) debugLog('DEBUG', 'auto-maintain', `boosted ${boosted.changes} frequently-accessed observations`);
+        // cleanup / decay+mark-idle / boost via maintain-core (shared with CLI + MCP).
+        // injection_count>0 protection lives in decayAndMarkIdle. Whole-DB, cap 500.
+        const mctx = { projectFilter: '', baseParams: [], staleAge: STALE_AGE, opCap: OP_CAP };
+
+        const cleaned = cleanupBroken(db, mctx);
+        if (cleaned > 0) debugLog('DEBUG', 'auto-maintain', `cleaned ${cleaned} broken observations`);
+
+        const { decayed, idleMarked } = decayAndMarkIdle(db, mctx);
+        if (decayed > 0) debugLog('DEBUG', 'auto-maintain', `decayed ${decayed} stale observations`);
+        if (idleMarked > 0) debugLog('DEBUG', 'auto-maintain', `marked ${idleMarked} idle as pending-purge`);
+
+        const boosted = boostAccessed(db, mctx);
+        if (boosted > 0) debugLog('DEBUG', 'auto-maintain', `boosted ${boosted} frequently-accessed observations`);
 
         // Auto-dedup (exact): merge identical-title observations within 1h.
         // Catches rapid duplicate writes (same hook firing twice, race conditions).
@@ -1361,57 +1317,16 @@ function handleAutoCompress() {
 
   try {
     const compressCutoff = Date.now() - 60 * 86400000; // 60 days
-    const compressCandidates = db.prepare(`
-      SELECT id, project, type, title, created_at_epoch
-      FROM observations
-      WHERE COALESCE(importance, 1) = 1 AND COALESCE(access_count, 0) = 0
-        AND created_at_epoch < ?
-        AND (compressed_into IS NULL OR compressed_into = ${COMPRESSED_AUTO})
-      ORDER BY project, created_at_epoch
-    `).all(compressCutoff);
+    const compressCandidates = selectCompressionCandidates(db, { cutoff: compressCutoff, includeAutoMarked: true });
     if (compressCandidates.length < 3) return;
 
-    const groups = new Map();
-    for (const c of compressCandidates) {
-      const key = `${c.project}::${isoWeekKey(c.created_at_epoch)}`;
-      if (!groups.has(key)) groups.set(key, []);
-      groups.get(key).push(c);
-    }
-    // Transact each group to prevent orphan summaries on crash
-    const compressGroup = db.transaction((proj, obs) => {
-      const types = {};
-      for (const o of obs) types[o.type] = (types[o.type] || 0) + 1;
-      const dominantType = Object.entries(types).sort((a, b) => b[1] - a[1])[0][0];
-      const title = `Weekly summary: ${obs.length} ${dominantType} observations`;
-      const narrative = obs.map(o => `- ${o.title || '(untitled)'}`).join('\n');
-      const sortedEpochs = obs.map(o => o.created_at_epoch).sort((a, b) => a - b);
-      const medianEpoch = sortedEpochs[Math.floor(sortedEpochs.length / 2)];
-      const sessionId = `compress-${proj}`;
-      const now = new Date();
-      db.prepare(`INSERT OR IGNORE INTO sdk_sessions
-        (content_session_id, memory_session_id, project, started_at, started_at_epoch, status)
-        VALUES (?,?,?,?,?,'active')`
-      ).run(sessionId, sessionId, proj, now.toISOString(), now.getTime());
-      // Defense-in-depth: title/narrative are derived from already-stored
-      // obs.title, but those rows pre-date the central scrub policy in some
-      // cases. Re-scrub at the persistence boundary.
-      const safe = scrubRecord('observations', { text: narrative, title, narrative });
-      const summaryResult = db.prepare(`INSERT INTO observations
-        (memory_session_id, project, text, type, title, subtitle, narrative, concepts, facts,
-         files_read, files_modified, importance, created_at, created_at_epoch)
-        VALUES (?,?,?,?,?,'',?,'','','[]','[]',2,?,?)`
-      ).run(sessionId, proj, safe.text, dominantType, safe.title, safe.narrative, new Date(medianEpoch).toISOString(), medianEpoch);
-      const summaryId = Number(summaryResult.lastInsertRowid);
-      const obsIds = obs.map(o => o.id);
-      db.prepare(`UPDATE observations SET compressed_into = ? WHERE id IN (${obsIds.map(() => '?').join(',')})`)
-        .run(summaryId, ...obsIds);
-      return obs.length;
-    });
+    const groups = groupByProjectWeek(compressCandidates);
+    // Transact each group to prevent orphan summaries on crash (CLI/MCP wrap all groups in one).
+    const compressGroupTxn = db.transaction((proj, obs) => compressGroup(db, proj, obs).compressed);
     let totalCompressed = 0;
     for (const [key, obs] of groups) {
-      if (obs.length < 3) continue;
       const [proj] = key.split('::');
-      totalCompressed += compressGroup(proj, obs);
+      totalCompressed += compressGroupTxn(proj, obs);
     }
     if (totalCompressed > 0) {
       debugLog('DEBUG', 'auto-compress', `auto-compressed ${totalCompressed} observations into weekly summaries`);

package/lib/compress-core.mjs

@@ -0,0 +1,98 @@
+// Shared "compress old low-value observations into weekly summaries" core.
+//
+// Single source of truth for cmdCompress (CLI), mem_compress (MCP), and
+// handleAutoCompress (hook). Pre-extraction the candidate query, the
+// project+ISO-week grouping, and the per-group summary INSERT + mark-compressed
+// were copy-pasted across all three and kept in sync by hand-written "parity"
+// comments — which is exactly how the TF-IDF-vector write drifted out of the
+// compression path (audit ARCH-1). Call sites keep what legitimately differs:
+// argument parsing, preview rendering, candidate-window params, and transaction
+// granularity (CLI/MCP wrap all groups in one transaction; the hook transacts
+// each group). They no longer re-implement the mutation.
+//
+// NOTE: the summary INSERT still omits the observation_vectors write, matching
+// pre-extraction behavior. Fixing that (audit P5) is now a single change here
+// instead of three — but it is a behavior change, intentionally NOT bundled.
+
+import { isoWeekKey, COMPRESSED_AUTO } from '../utils.mjs';
+import { scrubRecord } from './scrub-record.mjs';
+
+/**
+ * Low-value compression candidates: importance=1, never accessed, older than
+ * `cutoff`, not already compressed. `includeAutoMarked` also folds in rows the
+ * hook lightweight-marked as COMPRESSED_AUTO (the hook re-summarizes those).
+ */
+export function selectCompressionCandidates(db, { cutoff, project = null, includeAutoMarked = false }) {
+  const compressedFilter = includeAutoMarked
+    ? `AND (compressed_into IS NULL OR compressed_into = ${COMPRESSED_AUTO})`
+    : 'AND compressed_into IS NULL';
+  const projectFilter = project ? 'AND project = ?' : '';
+  const params = project ? [cutoff, project] : [cutoff];
+  return db.prepare(`
+    SELECT id, project, type, title, created_at, created_at_epoch
+    FROM observations
+    WHERE COALESCE(importance, 1) = 1
+      AND COALESCE(access_count, 0) = 0
+      AND created_at_epoch < ?
+      ${compressedFilter}
+      ${projectFilter}
+    ORDER BY project, created_at_epoch
+  `).all(...params);
+}
+
+/**
+ * Group candidates by `project::isoWeek` and keep only groups worth compressing
+ * (≥ 3 observations). Returns [[key, obs[]], …] — callers split the key on '::'
+ * for the project.
+ */
+export function groupByProjectWeek(candidates) {
+  const groups = new Map();
+  for (const c of candidates) {
+    const key = `${c.project}::${isoWeekKey(c.created_at_epoch)}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key).push(c);
+  }
+  return [...groups.entries()].filter(([, obs]) => obs.length >= 3);
+}
+
+/**
+ * Compress one group: create a weekly-summary observation (importance 2, dominant
+ * type, median timestamp so it sits correctly in recency/timeline), then mark all
+ * sources as compressed into it. Statement-only — the CALLER owns the transaction
+ * boundary (all-groups-in-one for CLI/MCP, per-group for the hook).
+ *
+ * @returns {{ summaryId: number, compressed: number }}
+ */
+export function compressGroup(db, proj, obs) {
+  const types = {};
+  for (const o of obs) types[o.type] = (types[o.type] || 0) + 1;
+  const dominantType = Object.entries(types).sort((a, b) => b[1] - a[1])[0][0];
+  const title = `Weekly summary: ${obs.length} ${dominantType} observations`;
+  const narrative = obs.map((o) => `- ${o.title || '(untitled)'}`).join('\n');
+  const sessionId = `compress-${proj}`;
+
+  const sortedEpochs = obs.map((o) => o.created_at_epoch).sort((a, b) => a - b);
+  const medianEpoch = sortedEpochs[Math.floor(sortedEpochs.length / 2)];
+  const medianDate = new Date(medianEpoch);
+
+  const now = new Date();
+  db.prepare(`
+    INSERT OR IGNORE INTO sdk_sessions (content_session_id, memory_session_id, project, started_at, started_at_epoch, status)
+    VALUES (?, ?, ?, ?, ?, 'active')
+  `).run(sessionId, sessionId, proj, now.toISOString(), now.getTime());
+
+  // Defense-in-depth: source rows were scrubbed at ingest, but the new narrative
+  // is constructed here and re-persisted.
+  const safe = scrubRecord('observations', { text: narrative, title, narrative });
+  const summaryResult = db.prepare(`
+    INSERT INTO observations (memory_session_id, project, text, type, title, subtitle, narrative, concepts, facts, files_read, files_modified, importance, created_at, created_at_epoch)
+    VALUES (?, ?, ?, ?, ?, '', ?, '', '', '[]', '[]', 2, ?, ?)
+  `).run(sessionId, proj, safe.text, dominantType, safe.title, safe.narrative, medianDate.toISOString(), medianEpoch);
+  const summaryId = Number(summaryResult.lastInsertRowid);
+
+  const obsIds = obs.map((o) => o.id);
+  const obsPh = obsIds.map(() => '?').join(',');
+  db.prepare(`UPDATE observations SET compressed_into = ? WHERE id IN (${obsPh})`).run(summaryId, ...obsIds);
+
+  return { summaryId, compressed: obs.length };
+}

package/lib/maintain-core.mjs

@@ -0,0 +1,236 @@
+// Shared maintenance operations — single source of truth for cmdMaintain (CLI),
+// mem_maintain (MCP), and handleAutoMaintain (hook). Pre-extraction each
+// operation's SQL was copy-pasted across the call sites and kept in sync by
+// "parity" comments, which had already drifted: the CLI/hook `decay` and
+// `mark-idle` protect injection_count>0 (v2.56.0 — an obs Claude was shown 8×
+// is contextually proven), but the MCP copy never got that clause, so
+// mem_maintain decayed/purged injected memories the other two paths preserve.
+// Consolidating here UNIFIES decay/mark-idle on the protected (correct) form.
+//
+// Every mutation is statement-only — the CALLER owns the transaction boundary
+// (CLI/MCP wrap the execute ops in one transaction; the hook runs them in its
+// auto-maintain block). `ctx` carries the per-caller knobs:
+//   { projectFilter: 'AND project = ?' | '', baseParams: [project?] , staleAge, opCap }
+
+import { COMPRESSED_PENDING_PURGE, computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity } from '../utils.mjs';
+import { rebuildVocabulary, computeVector, _resetVocabCache } from '../tfidf.mjs';
+
+export const STALE_AGE_MS = 30 * 86400000;
+export const OP_CAP = 1000;
+export const SCAN_LIMIT = 500;
+export const DUPLICATE_LIMIT = 50;
+export const SIMILARITY_THRESHOLD = 0.7;
+export const MINHASH_PRE_THRESHOLD = 0.5;
+// A memory injected this many times with zero citations is "pinned noise" that
+// the regular decay op can't touch (decay protects injection_count>0).
+export const PINNED_INJ_THRESHOLD = 8;
+
+/** Delete broken observations (no title AND no narrative). Returns rows deleted. */
+export function cleanupBroken(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    DELETE FROM observations WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND (title IS NULL OR title = '') AND (narrative IS NULL OR narrative = '')
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Decay importance of old, never-accessed, NEVER-INJECTED observations, then mark
+ * the importance-1 idle ones as pending-purge. injection_count>0 is protected as
+ * first-class engagement alongside access_count (unified across all three paths).
+ */
+export function decayAndMarkIdle(db, { projectFilter, baseParams, staleAge, opCap = OP_CAP }) {
+  const decayed = db.prepare(`
+    UPDATE observations SET importance = MAX(1, COALESCE(importance, 1) - 1)
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(importance, 1) > 1
+        AND COALESCE(access_count, 0) = 0
+        AND COALESCE(injection_count, 0) = 0
+        AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(staleAge, ...baseParams).changes;
+
+  const idleMarked = db.prepare(`
+    UPDATE observations SET compressed_into = ${COMPRESSED_PENDING_PURGE}
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(importance, 1) = 1
+        AND COALESCE(access_count, 0) = 0
+        AND COALESCE(injection_count, 0) = 0
+        AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(staleAge, ...baseParams).changes;
+
+  return { decayed, idleMarked };
+}
+
+/** Boost importance of frequently-accessed observations. Returns rows boosted. */
+export function boostAccessed(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    UPDATE observations SET importance = MIN(3, COALESCE(importance, 1) + 1)
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(access_count, 0) > 3
+        AND COALESCE(importance, 1) < 3
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Repair the citation-decay blind spot: heavy-injection + zero-citation rows that
+ * decay protects (injection_count>0) stay pinned at max importance forever. Drop
+ * them to importance 1 in one pass (injection priority is binary at >=2, so a
+ * single step would not de-rank). Floor 1, not purge.
+ */
+export function demotePinned(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    UPDATE observations SET importance = 1
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(injection_count, 0) >= ${PINNED_INJ_THRESHOLD}
+        AND COALESCE(cited_count, 0) = 0
+        AND COALESCE(importance, 1) > 1
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Merge explicit duplicate groups: each group is [keepId, removeId, …]. Marks the
+ * removeIds compressed into keepId (only if not already compressed). Returns the
+ * number of rows merged. Callers parse their own input (CLI string / MCP array).
+ */
+export function mergeDuplicates(db, groups) {
+  let merged = 0;
+  const mergeStmt = db.prepare('UPDATE observations SET compressed_into = ? WHERE id = ? AND COALESCE(compressed_into, 0) = 0');
+  for (const group of groups) {
+    if (!group || group.length < 2) continue;
+    const [keepId, ...removeIds] = group;
+    for (const removeId of removeIds) merged += mergeStmt.run(keepId, removeId).changes;
+  }
+  return merged;
+}
+
+/** Preview pending-purge candidates older than the retain cutoff (no deletion). */
+export function purgeStalePreview(db, { projectFilter, baseParams }, retainCutoff) {
+  return db.prepare(`
+    SELECT COUNT(*) AS candidates, MIN(created_at_epoch) AS oldest, MAX(created_at_epoch) AS newest
+    FROM observations
+    WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} AND created_at_epoch < ? ${projectFilter}
+  `).get(retainCutoff, ...baseParams);
+}
+
+/** Delete pending-purge observations older than the retain cutoff. Returns rows deleted. */
+export function purgeStale(db, { projectFilter, baseParams, opCap = OP_CAP }, retainCutoff) {
+  return db.prepare(`
+    DELETE FROM observations WHERE id IN (
+      SELECT id FROM observations
+      WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(retainCutoff, ...baseParams).changes;
+}
+
+/**
+ * Near-duplicate title detection: MinHash pre-filter → exact Jaccard. Returns
+ * [{ a:{id,title,importance}, b:{…}, similarity:'0.NN' }, …].
+ */
+export function findDuplicates(db, { projectFilter, baseParams, limit = SCAN_LIMIT, dupLimit = DUPLICATE_LIMIT }) {
+  const recent = db.prepare(`
+    SELECT id, title, project, importance, access_count, created_at_epoch
+    FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}
+    ORDER BY created_at_epoch DESC LIMIT ${limit}
+  `).all(...baseParams);
+
+  const titles = recent.map((r) => (r.title || '').trim());
+  const minhashes = titles.map((t) => (t ? computeMinHash(t) : null));
+  const duplicates = [];
+  for (let i = 0; i < recent.length && duplicates.length < dupLimit; i++) {
+    if (!titles[i] || !minhashes[i]) continue;
+    for (let j = i + 1; j < recent.length; j++) {
+      if (!titles[j] || !minhashes[j]) continue;
+      if (estimateJaccardFromMinHash(minhashes[i], minhashes[j]) < MINHASH_PRE_THRESHOLD) continue;
+      const sim = jaccardSimilarity(titles[i], titles[j]);
+      if (sim > SIMILARITY_THRESHOLD) {
+        duplicates.push({
+          a: { id: recent[i].id, title: recent[i].title, importance: recent[i].importance },
+          b: { id: recent[j].id, title: recent[j].title, importance: recent[j].importance },
+          similarity: sim.toFixed(2),
+        });
+      }
+      if (duplicates.length >= dupLimit) break;
+    }
+  }
+  return duplicates;
+}
+
+/** Single-scan maintenance counters (includes `pinned`; callers render what they show). */
+export function maintenanceStats(db, { projectFilter, baseParams, staleAge }) {
+  const stats = db.prepare(`
+    SELECT
+      COUNT(*) as total,
+      COALESCE(SUM(CASE WHEN COALESCE(importance, 1) = 1 AND COALESCE(access_count, 0) = 0
+                    AND created_at_epoch < ? THEN 1 ELSE 0 END), 0) as stale,
+      COALESCE(SUM(CASE WHEN (title IS NULL OR title = '') AND (narrative IS NULL OR narrative = '')
+               THEN 1 ELSE 0 END), 0) as broken,
+      COALESCE(SUM(CASE WHEN COALESCE(access_count, 0) > 3 AND COALESCE(importance, 1) < 3
+               THEN 1 ELSE 0 END), 0) as boostable,
+      COALESCE(SUM(CASE WHEN COALESCE(injection_count, 0) >= ${PINNED_INJ_THRESHOLD}
+                    AND COALESCE(cited_count, 0) = 0 AND COALESCE(importance, 1) > 1
+               THEN 1 ELSE 0 END), 0) as pinned
+    FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}
+  `).get(staleAge, ...baseParams);
+  const pendingPurge = db.prepare(
+    `SELECT COUNT(*) as count FROM observations WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} ${projectFilter}`
+  ).get(...baseParams);
+  return { ...stats, pendingPurge: pendingPurge.count };
+}
+
+/** Rebuild the TF-IDF vocabulary + every active observation vector (own transaction). */
+export function rebuildVectors(db) {
+  _resetVocabCache();
+  const vocab = rebuildVocabulary(db);
+  if (!vocab) return { ok: false, reason: 'no observations to build vocabulary from' };
+  const allObs = db.prepare(`
+    SELECT id, title, narrative, concepts FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 AND superseded_at IS NULL
+  `).all();
+  let updated = 0;
+  const insertStmt = db.prepare('INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch) VALUES (?, ?, ?, ?)');
+  const now = Date.now();
+  db.transaction(() => {
+    db.prepare('DELETE FROM observation_vectors').run();
+    for (const obs of allObs) {
+      const text = [obs.title || '', obs.narrative || '', obs.concepts || ''].filter(Boolean).join(' ');
+      const vec = computeVector(text, vocab);
+      if (vec) {
+        insertStmt.run(obs.id, Buffer.from(vec.buffer), vocab.version, now);
+        updated++;
+      }
+    }
+  })();
+  return { ok: true, terms: vocab.terms.size, updated, total: allObs.length };
+}
+
+/** VACUUM the whole DB, reporting freelist reclaim. Must run OUTSIDE any transaction. */
+export function vacuum(db) {
+  const pageSize = db.pragma('page_size', { simple: true });
+  const freeBefore = db.pragma('freelist_count', { simple: true });
+  db.exec('VACUUM');
+  const freeAfter = db.pragma('freelist_count', { simple: true });
+  const reclaimedMB = ((Math.max(0, freeBefore - freeAfter) * pageSize) / 1048576).toFixed(1);
+  return { reclaimedMB, freeBefore, freeAfter };
+}