claude-mem-lite 2.87.0 → 2.89.0

package/lib/compress-core.mjs

@@ -0,0 +1,118 @@
+// Shared "compress old low-value observations into weekly summaries" core.
+//
+// Single source of truth for cmdCompress (CLI), mem_compress (MCP), and
+// handleAutoCompress (hook). Pre-extraction the candidate query, the
+// project+ISO-week grouping, and the per-group summary INSERT + mark-compressed
+// were copy-pasted across all three and kept in sync by hand-written "parity"
+// comments — which is exactly how the TF-IDF-vector write drifted out of the
+// compression path (audit ARCH-1). Call sites keep what legitimately differs:
+// argument parsing, preview rendering, candidate-window params, and transaction
+// granularity (CLI/MCP wrap all groups in one transaction; the hook transacts
+// each group). They no longer re-implement the mutation.
+//
+// The summary INSERT also writes its TF-IDF observation_vectors row in the same
+// (caller-owned) transaction — fixed once here rather than in all three call
+// sites. Without it, FTS-miss queries that fall back to vector recall (CJK /
+// concept / paraphrase) could never reach compressed summaries; the LLM
+// smart-compress path already wrote vectors, so the deterministic path was the
+// sole gap (audit P6).
+
+import { isoWeekKey, COMPRESSED_AUTO, debugCatch } from '../utils.mjs';
+import { getVocabulary, computeVector } from '../tfidf.mjs';
+import { scrubRecord } from './scrub-record.mjs';
+
+/**
+ * Low-value compression candidates: importance=1, never accessed, older than
+ * `cutoff`, not already compressed. `includeAutoMarked` also folds in rows the
+ * hook lightweight-marked as COMPRESSED_AUTO (the hook re-summarizes those).
+ */
+export function selectCompressionCandidates(db, { cutoff, project = null, includeAutoMarked = false }) {
+  const compressedFilter = includeAutoMarked
+    ? `AND (compressed_into IS NULL OR compressed_into = ${COMPRESSED_AUTO})`
+    : 'AND compressed_into IS NULL';
+  const projectFilter = project ? 'AND project = ?' : '';
+  const params = project ? [cutoff, project] : [cutoff];
+  return db.prepare(`
+    SELECT id, project, type, title, created_at, created_at_epoch
+    FROM observations
+    WHERE COALESCE(importance, 1) = 1
+      AND COALESCE(access_count, 0) = 0
+      AND created_at_epoch < ?
+      ${compressedFilter}
+      ${projectFilter}
+    ORDER BY project, created_at_epoch
+  `).all(...params);
+}
+
+/**
+ * Group candidates by `project::isoWeek` and keep only groups worth compressing
+ * (≥ 3 observations). Returns [[key, obs[]], …] — callers split the key on '::'
+ * for the project.
+ */
+export function groupByProjectWeek(candidates) {
+  const groups = new Map();
+  for (const c of candidates) {
+    const key = `${c.project}::${isoWeekKey(c.created_at_epoch)}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key).push(c);
+  }
+  return [...groups.entries()].filter(([, obs]) => obs.length >= 3);
+}
+
+/**
+ * Compress one group: create a weekly-summary observation (importance 2, dominant
+ * type, median timestamp so it sits correctly in recency/timeline), then mark all
+ * sources as compressed into it. Statement-only — the CALLER owns the transaction
+ * boundary (all-groups-in-one for CLI/MCP, per-group for the hook).
+ *
+ * @returns {{ summaryId: number, compressed: number }}
+ */
+export function compressGroup(db, proj, obs) {
+  const types = {};
+  for (const o of obs) types[o.type] = (types[o.type] || 0) + 1;
+  const dominantType = Object.entries(types).sort((a, b) => b[1] - a[1])[0][0];
+  const title = `Weekly summary: ${obs.length} ${dominantType} observations`;
+  const narrative = obs.map((o) => `- ${o.title || '(untitled)'}`).join('\n');
+  const sessionId = `compress-${proj}`;
+
+  const sortedEpochs = obs.map((o) => o.created_at_epoch).sort((a, b) => a - b);
+  const medianEpoch = sortedEpochs[Math.floor(sortedEpochs.length / 2)];
+  const medianDate = new Date(medianEpoch);
+
+  const now = new Date();
+  db.prepare(`
+    INSERT OR IGNORE INTO sdk_sessions (content_session_id, memory_session_id, project, started_at, started_at_epoch, status)
+    VALUES (?, ?, ?, ?, ?, 'active')
+  `).run(sessionId, sessionId, proj, now.toISOString(), now.getTime());
+
+  // Defense-in-depth: source rows were scrubbed at ingest, but the new narrative
+  // is constructed here and re-persisted.
+  const safe = scrubRecord('observations', { text: narrative, title, narrative });
+  const summaryResult = db.prepare(`
+    INSERT INTO observations (memory_session_id, project, text, type, title, subtitle, narrative, concepts, facts, files_read, files_modified, importance, created_at, created_at_epoch)
+    VALUES (?, ?, ?, ?, ?, '', ?, '', '', '[]', '[]', 2, ?, ?)
+  `).run(sessionId, proj, safe.text, dominantType, safe.title, safe.narrative, medianDate.toISOString(), medianEpoch);
+  const summaryId = Number(summaryResult.lastInsertRowid);
+
+  // TF-IDF vector for the summary so it is reachable by vector recall (parity
+  // with save-observation.mjs and the LLM smart-compress path). Best-effort:
+  // vocab may be uninitialized on a fresh DB — a failure here must not abort the
+  // compression the caller is transacting.
+  try {
+    const vocab = getVocabulary(db);
+    if (vocab) {
+      const vec = computeVector(`${safe.title} ${safe.narrative}`, vocab);
+      if (vec) {
+        db.prepare(
+          'INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch) VALUES (?, ?, ?, ?)'
+        ).run(summaryId, Buffer.from(vec.buffer), vocab.version, medianEpoch);
+      }
+    }
+  } catch (e) { debugCatch(e, 'compress-vector'); }
+
+  const obsIds = obs.map((o) => o.id);
+  const obsPh = obsIds.map(() => '?').join(',');
+  db.prepare(`UPDATE observations SET compressed_into = ? WHERE id IN (${obsPh})`).run(summaryId, ...obsIds);
+
+  return { summaryId, compressed: obs.length };
+}

package/lib/dedup-constants.mjs

@@ -0,0 +1,35 @@
+// Dedup / merge similarity thresholds — single source of truth (P10).
+//
+// All values are Jaccard-space (word-set overlap, 0..1) unless noted. They were
+// scattered as bare literals and duplicate local consts across save-observation,
+// maintain-core, hook-llm, hook-optimize, mem-cli, server, and hook; converging
+// them here removes the drift risk and gives the P7 benchmark named knobs.
+// Vector-side constants (VOCAB_DIM / MIN_COSINE_SIMILARITY / RRF_K) deliberately
+// stay in tfidf.mjs next to the search engine that consumes them.
+//
+// Pure constants only — no imports, so nothing can import-cycle through this.
+
+// 0.7: near-duplicate cutoff for save-time dedup (5-min window, lib/save-observation)
+// and the hook-llm tier-1 title dedup. Catches "Modified X" / "Fixed X" restatements
+// (~70% word overlap) without collapsing distinct-but-related observations.
+export const DEDUP_JACCARD_THRESHOLD = 0.7;
+
+// 0.85: high-confidence auto-merge cutoff (maintain + optimize cluster-merge, CLI/MCP
+// dedup preview). Pairs at or above this merge without an LLM merge-decision call.
+export const AUTO_MERGE_THRESHOLD = 0.85;
+
+// 0.4: low bound of the LLM-review merge band [0.4, 0.85) in hook-optimize. Below it,
+// a pair is too dissimilar to be worth a merge-decision call.
+export const MERGE_JACCARD_LOW = 0.4;
+
+// 0.5: MinHash estimated-Jaccard pre-filter for the maintain O(n²) scan — skip the
+// exact-Jaccard compare when the cheap signature estimate is already below this.
+export const MINHASH_PRE_THRESHOLD = 0.5;
+
+// 0.7: MinHash pre-filter for the hook post-inject fuzzy-dedup pass. Stricter than
+// maintain's 0.5 to keep the inline inject path cheap (it runs in the hot Stop path).
+export const MINHASH_PREFILTER = 0.7;
+
+// 0.95: strict title-Jaccard cutoff for the hook post-inject fuzzy-dedup pass — only
+// collapse near-identical titles inline; anything softer waits for the maintain sweep.
+export const FUZZY_DEDUP_THRESHOLD = 0.95;

package/lib/maintain-core.mjs

@@ -0,0 +1,239 @@
+// Shared maintenance operations — single source of truth for cmdMaintain (CLI),
+// mem_maintain (MCP), and handleAutoMaintain (hook). Pre-extraction each
+// operation's SQL was copy-pasted across the call sites and kept in sync by
+// "parity" comments, which had already drifted: the CLI/hook `decay` and
+// `mark-idle` protect injection_count>0 (v2.56.0 — an obs Claude was shown 8×
+// is contextually proven), but the MCP copy never got that clause, so
+// mem_maintain decayed/purged injected memories the other two paths preserve.
+// Consolidating here UNIFIES decay/mark-idle on the protected (correct) form.
+//
+// Every mutation is statement-only — the CALLER owns the transaction boundary
+// (CLI/MCP wrap the execute ops in one transaction; the hook runs them in its
+// auto-maintain block). `ctx` carries the per-caller knobs:
+//   { projectFilter: 'AND project = ?' | '', baseParams: [project?] , staleAge, opCap }
+
+import { COMPRESSED_PENDING_PURGE, computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity } from '../utils.mjs';
+import { rebuildVocabulary, computeVector, _resetVocabCache } from '../tfidf.mjs';
+import { DEDUP_JACCARD_THRESHOLD, MINHASH_PRE_THRESHOLD as MINHASH_PRE_THRESHOLD_SRC } from './dedup-constants.mjs';
+
+export const STALE_AGE_MS = 30 * 86400000;
+export const OP_CAP = 1000;
+export const SCAN_LIMIT = 500;
+export const DUPLICATE_LIMIT = 50;
+// Back-compat: maintain-core historically exported these names; both now source
+// their value from the single canonical lib/dedup-constants.mjs.
+export const SIMILARITY_THRESHOLD = DEDUP_JACCARD_THRESHOLD;
+export const MINHASH_PRE_THRESHOLD = MINHASH_PRE_THRESHOLD_SRC;
+// A memory injected this many times with zero citations is "pinned noise" that
+// the regular decay op can't touch (decay protects injection_count>0).
+export const PINNED_INJ_THRESHOLD = 8;
+
+/** Delete broken observations (no title AND no narrative). Returns rows deleted. */
+export function cleanupBroken(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    DELETE FROM observations WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND (title IS NULL OR title = '') AND (narrative IS NULL OR narrative = '')
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Decay importance of old, never-accessed, NEVER-INJECTED observations, then mark
+ * the importance-1 idle ones as pending-purge. injection_count>0 is protected as
+ * first-class engagement alongside access_count (unified across all three paths).
+ */
+export function decayAndMarkIdle(db, { projectFilter, baseParams, staleAge, opCap = OP_CAP }) {
+  const decayed = db.prepare(`
+    UPDATE observations SET importance = MAX(1, COALESCE(importance, 1) - 1)
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(importance, 1) > 1
+        AND COALESCE(access_count, 0) = 0
+        AND COALESCE(injection_count, 0) = 0
+        AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(staleAge, ...baseParams).changes;
+
+  const idleMarked = db.prepare(`
+    UPDATE observations SET compressed_into = ${COMPRESSED_PENDING_PURGE}
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(importance, 1) = 1
+        AND COALESCE(access_count, 0) = 0
+        AND COALESCE(injection_count, 0) = 0
+        AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(staleAge, ...baseParams).changes;
+
+  return { decayed, idleMarked };
+}
+
+/** Boost importance of frequently-accessed observations. Returns rows boosted. */
+export function boostAccessed(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    UPDATE observations SET importance = MIN(3, COALESCE(importance, 1) + 1)
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(access_count, 0) > 3
+        AND COALESCE(importance, 1) < 3
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Repair the citation-decay blind spot: heavy-injection + zero-citation rows that
+ * decay protects (injection_count>0) stay pinned at max importance forever. Drop
+ * them to importance 1 in one pass (injection priority is binary at >=2, so a
+ * single step would not de-rank). Floor 1, not purge.
+ */
+export function demotePinned(db, { projectFilter, baseParams, opCap = OP_CAP }) {
+  return db.prepare(`
+    UPDATE observations SET importance = 1
+    WHERE id IN (
+      SELECT id FROM observations
+      WHERE COALESCE(compressed_into, 0) = 0
+        AND COALESCE(injection_count, 0) >= ${PINNED_INJ_THRESHOLD}
+        AND COALESCE(cited_count, 0) = 0
+        AND COALESCE(importance, 1) > 1
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(...baseParams).changes;
+}
+
+/**
+ * Merge explicit duplicate groups: each group is [keepId, removeId, …]. Marks the
+ * removeIds compressed into keepId (only if not already compressed). Returns the
+ * number of rows merged. Callers parse their own input (CLI string / MCP array).
+ */
+export function mergeDuplicates(db, groups) {
+  let merged = 0;
+  const mergeStmt = db.prepare('UPDATE observations SET compressed_into = ? WHERE id = ? AND COALESCE(compressed_into, 0) = 0');
+  for (const group of groups) {
+    if (!group || group.length < 2) continue;
+    const [keepId, ...removeIds] = group;
+    for (const removeId of removeIds) merged += mergeStmt.run(keepId, removeId).changes;
+  }
+  return merged;
+}
+
+/** Preview pending-purge candidates older than the retain cutoff (no deletion). */
+export function purgeStalePreview(db, { projectFilter, baseParams }, retainCutoff) {
+  return db.prepare(`
+    SELECT COUNT(*) AS candidates, MIN(created_at_epoch) AS oldest, MAX(created_at_epoch) AS newest
+    FROM observations
+    WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} AND created_at_epoch < ? ${projectFilter}
+  `).get(retainCutoff, ...baseParams);
+}
+
+/** Delete pending-purge observations older than the retain cutoff. Returns rows deleted. */
+export function purgeStale(db, { projectFilter, baseParams, opCap = OP_CAP }, retainCutoff) {
+  return db.prepare(`
+    DELETE FROM observations WHERE id IN (
+      SELECT id FROM observations
+      WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} AND created_at_epoch < ?
+        ${projectFilter} LIMIT ${opCap}
+    )
+  `).run(retainCutoff, ...baseParams).changes;
+}
+
+/**
+ * Near-duplicate title detection: MinHash pre-filter → exact Jaccard. Returns
+ * [{ a:{id,title,importance}, b:{…}, similarity:'0.NN' }, …].
+ */
+export function findDuplicates(db, { projectFilter, baseParams, limit = SCAN_LIMIT, dupLimit = DUPLICATE_LIMIT }) {
+  const recent = db.prepare(`
+    SELECT id, title, project, importance, access_count, created_at_epoch
+    FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}
+    ORDER BY created_at_epoch DESC LIMIT ${limit}
+  `).all(...baseParams);
+
+  const titles = recent.map((r) => (r.title || '').trim());
+  const minhashes = titles.map((t) => (t ? computeMinHash(t) : null));
+  const duplicates = [];
+  for (let i = 0; i < recent.length && duplicates.length < dupLimit; i++) {
+    if (!titles[i] || !minhashes[i]) continue;
+    for (let j = i + 1; j < recent.length; j++) {
+      if (!titles[j] || !minhashes[j]) continue;
+      if (estimateJaccardFromMinHash(minhashes[i], minhashes[j]) < MINHASH_PRE_THRESHOLD) continue;
+      const sim = jaccardSimilarity(titles[i], titles[j]);
+      if (sim > SIMILARITY_THRESHOLD) {
+        duplicates.push({
+          a: { id: recent[i].id, title: recent[i].title, importance: recent[i].importance },
+          b: { id: recent[j].id, title: recent[j].title, importance: recent[j].importance },
+          similarity: sim.toFixed(2),
+        });
+      }
+      if (duplicates.length >= dupLimit) break;
+    }
+  }
+  return duplicates;
+}
+
+/** Single-scan maintenance counters (includes `pinned`; callers render what they show). */
+export function maintenanceStats(db, { projectFilter, baseParams, staleAge }) {
+  const stats = db.prepare(`
+    SELECT
+      COUNT(*) as total,
+      COALESCE(SUM(CASE WHEN COALESCE(importance, 1) = 1 AND COALESCE(access_count, 0) = 0
+                    AND created_at_epoch < ? THEN 1 ELSE 0 END), 0) as stale,
+      COALESCE(SUM(CASE WHEN (title IS NULL OR title = '') AND (narrative IS NULL OR narrative = '')
+               THEN 1 ELSE 0 END), 0) as broken,
+      COALESCE(SUM(CASE WHEN COALESCE(access_count, 0) > 3 AND COALESCE(importance, 1) < 3
+               THEN 1 ELSE 0 END), 0) as boostable,
+      COALESCE(SUM(CASE WHEN COALESCE(injection_count, 0) >= ${PINNED_INJ_THRESHOLD}
+                    AND COALESCE(cited_count, 0) = 0 AND COALESCE(importance, 1) > 1
+               THEN 1 ELSE 0 END), 0) as pinned
+    FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 ${projectFilter}
+  `).get(staleAge, ...baseParams);
+  const pendingPurge = db.prepare(
+    `SELECT COUNT(*) as count FROM observations WHERE compressed_into = ${COMPRESSED_PENDING_PURGE} ${projectFilter}`
+  ).get(...baseParams);
+  return { ...stats, pendingPurge: pendingPurge.count };
+}
+
+/** Rebuild the TF-IDF vocabulary + every active observation vector (own transaction). */
+export function rebuildVectors(db) {
+  _resetVocabCache();
+  const vocab = rebuildVocabulary(db);
+  if (!vocab) return { ok: false, reason: 'no observations to build vocabulary from' };
+  const allObs = db.prepare(`
+    SELECT id, title, narrative, concepts FROM observations
+    WHERE COALESCE(compressed_into, 0) = 0 AND superseded_at IS NULL
+  `).all();
+  let updated = 0;
+  const insertStmt = db.prepare('INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch) VALUES (?, ?, ?, ?)');
+  const now = Date.now();
+  db.transaction(() => {
+    db.prepare('DELETE FROM observation_vectors').run();
+    for (const obs of allObs) {
+      const text = [obs.title || '', obs.narrative || '', obs.concepts || ''].filter(Boolean).join(' ');
+      const vec = computeVector(text, vocab);
+      if (vec) {
+        insertStmt.run(obs.id, Buffer.from(vec.buffer), vocab.version, now);
+        updated++;
+      }
+    }
+  })();
+  return { ok: true, terms: vocab.terms.size, updated, total: allObs.length };
+}
+
+/** VACUUM the whole DB, reporting freelist reclaim. Must run OUTSIDE any transaction. */
+export function vacuum(db) {
+  const pageSize = db.pragma('page_size', { simple: true });
+  const freeBefore = db.pragma('freelist_count', { simple: true });
+  db.exec('VACUUM');
+  const freeAfter = db.pragma('freelist_count', { simple: true });
+  const reclaimedMB = ((Math.max(0, freeBefore - freeAfter) * pageSize) / 1048576).toFixed(1);
+  return { reclaimedMB, freeBefore, freeAfter };
+}

package/lib/save-observation.mjs

@@ -13,10 +13,10 @@
 
 import { jaccardSimilarity, scrubSecrets, computeMinHash, cjkBigrams, getCurrentBranch, debugCatch } from '../utils.mjs';
 import { getVocabulary, computeVector } from '../tfidf.mjs';
+import { DEDUP_JACCARD_THRESHOLD } from './dedup-constants.mjs';
 
 const DEDUP_WINDOW_MS = 5 * 60 * 1000;
 const DEDUP_RECENT_LIMIT = 50;
-const DEDUP_JACCARD_THRESHOLD = 0.7;
 
 /**
  * Save a new observation if it isn't a near-duplicate of one saved within the