claude-mem-lite 2.46.0 → 2.47.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.46.0",
+      "version": "2.47.0",
       "source": "./",
       "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.46.0",
+  "version": "2.47.0",
   "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
   "author": {
     "name": "sdsrss"

package/hook-llm.mjs

@@ -16,7 +16,7 @@ import {
   sessionFile, getSessionId, openDb, callLLM, sleep,
 } from './hook-shared.mjs';
 import { EVENT_TYPES, saveEvent } from './lib/activity.mjs';
-import { isNoiseObservation } from './lib/low-signal-patterns.mjs';
+import { isNoiseObservation, capNoiseImportance } from './lib/low-signal-patterns.mjs';
 
 // T9: memdir-incompatible types live in the `events` table, not `observations`.
 // Set lookup is O(1) — authoritative source is lib/activity.mjs::EVENT_TYPES.
@@ -78,6 +78,16 @@ export function saveObservation(obs, projectOverride, sessionIdOverride, externa
       return null;
     }
 
+    // v2.47 P0-3: importance cap for LOW_SIGNAL titles that kept the drop gate
+    // open via importance>=2 but carry no lesson/facts signal. 341 rows in live
+    // DB had imp=3 under these conditions (99.4% noise). Cap to 1 so they
+    // enter the 7-day accelerated auto-compress window in hook.mjs.
+    const capped = capNoiseImportance(obs);
+    if (capped !== (obs.importance ?? 1)) {
+      debugLog('saveObservation', `capped imp ${obs.importance}→${capped}: ${truncate(obs.title || '', 60)}`);
+      obs.importance = capped;
+    }
+
     // Three-tier dedup — returns null (not throw) for dedup hits
     // Tier 1 (fast): 5-min Jaccard on titles
     const fiveMinAgo = now.getTime() - DEDUP_WINDOW_MS;

package/hook.mjs

@@ -648,6 +648,29 @@ async function handleSessionStart() {
       if (compressed.changes > 0) {
         debugLog('DEBUG', 'session-start', `auto-compressed ${compressed.changes} old observations`);
       }
+
+      // v2.47 P0-3: accelerated compress for LOW_SIGNAL + no-signal noise.
+      // 7-day window instead of 30. The write-side capNoiseImportance forces
+      // imp=1 on these already; this just shrinks the GC latency so the
+      // projected 32.5% corpus reduction materializes within a week on live
+      // DBs instead of bleeding into the 30-day tier.
+      const noiseCompressAge = Date.now() - 7 * 86400000;
+      const noiseCompressed = db.prepare(`
+        UPDATE observations SET compressed_into = ${COMPRESSED_AUTO}
+        WHERE COALESCE(compressed_into, 0) = 0
+          AND importance = 1
+          AND (lesson_learned IS NULL OR lesson_learned = '' OR lesson_learned = 'none')
+          AND (facts IS NULL OR facts = '' OR facts = '[]')
+          AND (
+            title LIKE 'Modified %' OR title LIKE 'Worked on %'
+            OR title LIKE 'Reviewed %' OR title LIKE 'Error%'
+          )
+          AND created_at_epoch < ?
+          AND project = ?
+      `).run(noiseCompressAge, project);
+      if (noiseCompressed.changes > 0) {
+        debugLog('DEBUG', 'session-start', `auto-compressed ${noiseCompressed.changes} LOW_SIGNAL noise (7d window)`);
+      }
     })();
 
     // Auto-maintain: cleanup + decay + boost + purge, gated to once per 24h

package/lib/low-signal-patterns.mjs

@@ -113,6 +113,40 @@ function _isLikelyToolOutputPassthrough(narrative) {
  * @param {object} [env=process.env] Environment (injected for testability)
  * @returns {boolean} true = noise, caller should drop
  */
+/**
+ * v2.47 P0-3: Importance cap for LOW_SIGNAL titles that slipped through with
+ * inflated importance. Complements isNoiseObservation — that one drops rows
+ * entirely when narrative is also thin; this one keeps the row (useful for
+ * session history) but demotes the importance so injection ranking and
+ * auto-compress treat it as the noise it is.
+ *
+ * Production baseline (2026-04-24, projects--mem DB, 3789 obs):
+ *   LOW_SIGNAL title + importance=3 → 341 rows; only 1 had lesson, 1 had facts
+ *   LOW_SIGNAL title + importance=2 → 80 rows;  only 5 had lesson, 6 had facts
+ * 99%+ of those were Haiku-inflated noise. Cap forces imp=1 and the 7-day
+ * accelerated auto-compress in hook.mjs GCs them.
+ *
+ * Preserves importance when ANY real signal exists:
+ *   - lesson_learned (or camelCase lessonLearned) set and not 'none'
+ *   - facts array has >=1 non-empty string
+ * Non-LOW_SIGNAL titles are never capped (substantive prose is trusted).
+ *
+ * @param {object} obs { title, facts, importance, lesson_learned|lessonLearned }
+ * @returns {number} Capped importance (1 if LOW_SIGNAL+no-signal, else original)
+ */
+export function capNoiseImportance(obs) {
+  const original = obs?.importance ?? 1;
+  const title = (obs && obs.title) || '';
+  if (!_LOW_SIG_RE.test(title)) return original;
+  const lesson = obs.lessonLearned ?? obs.lesson_learned;
+  if (lesson && String(lesson).trim() && String(lesson).trim().toLowerCase() !== 'none') return original;
+  if (Array.isArray(obs.facts) &&
+      obs.facts.filter(f => typeof f === 'string' && f.trim().length > 0).length >= 1) {
+    return original;
+  }
+  return original > 1 ? 1 : original;
+}
+
 export function isNoiseObservation(obs, env = process.env) {
   if (env && env.CLAUDE_MEM_KEEP_LOW_SIGNAL === '1') return false;
   const title = (obs && obs.title) || '';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.46.0",
+  "version": "2.47.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "engines": {

package/schema.mjs

@@ -21,7 +21,12 @@ export const REGISTRY_DB_PATH = join(DB_DIR, 'resource-registry.db');
 // FTS delete+reinsert cycles and amplified SQLITE_CORRUPT_VTAB blast radius
 // (project_non_obvious.md). Migration drops the old triggers once and lets
 // ensureFTS recreate them with the scoped form.
-export const CURRENT_SCHEMA_VERSION = 27;
+//
+// v28 (v2.47): observation_vectors orphan + stale-vocab cleanup. Live DBs had
+// 2839/6429 (44%) orphaned rows (historic deletes during FK-OFF migrations)
+// and 3282/6429 (51%) stale-vocab rows (rebuildVocabulary never pruned old
+// versions before v2.47). Idempotent one-shot DELETE on ensureDb.
+export const CURRENT_SCHEMA_VERSION = 28;
 
 const CORE_SCHEMA = `
   CREATE TABLE IF NOT EXISTS sdk_sessions (
@@ -402,6 +407,17 @@ export function initSchema(db) {
 
   db.exec(`CREATE INDEX IF NOT EXISTS idx_obs_vectors_version ON observation_vectors(vocab_version)`);
 
+  // v28 (v2.47) P0-1: one-shot cleanup of orphaned observation_vectors.
+  // Live DBs accumulated 44% orphans even with ON DELETE CASCADE because
+  // early migrations ran with `foreign_keys=OFF` and deletes skipped cascade.
+  // Idempotent (NOT IN is empty on a clean DB), runs once per ensureDb().
+  try {
+    db.prepare(`
+      DELETE FROM observation_vectors
+      WHERE observation_id NOT IN (SELECT id FROM observations)
+    `).run();
+  } catch { /* non-critical — table-missing path handled by earlier CREATE */ }
+
   // Persisted vocabulary for stable TF-IDF vector indexing
   db.exec(`
     CREATE TABLE IF NOT EXISTS vocab_state (

package/scoring-sql.mjs

@@ -70,12 +70,17 @@ export const TYPE_QUALITY_CASE = `(
  *   - injection_count: bumped ONLY on UserPromptSubmit / hook-memory auto-inject
  *   - access_count: bumped on citation (c039352 P4), explicit recall, get, timeline
  *
- * Empirical thresholds (see docs/p0-injection-noise-baseline.txt, 53 transcripts):
- *   • High-noise legitimate use (#5597 29/10=2.9x): kept at 1.0× (below tier-1)
- *   • Moderate noise (#4352 44/9=4.89x): drops to 0.5× (tier-1 hit)
- *   • Pure noise (#4046 14/0=inf): drops to 0.5× (tier-1; count≥10 gate protects
- *     cold-start obs with legitimately no cites yet)
- *   • Entrenched noise (≥20 inject, ≥5× ratio): drops to 0.2× (tier-2)
+ * Empirical thresholds (v2.47 recalibration — 2026-04-24 live projects--mem,
+ * 3789 obs, baseline 10/20 never fired because max injection_count=9):
+ *   • Legitimate heavy use (#5588 9/10=0.9, #7549 7/13=0.54): ratio≤3 ⇒ 1.0×
+ *   • Early noise candidate (#3518 6/1=6.0): inj≥4 AND ratio>3 ⇒ 0.5× (tier-1)
+ *   • Entrenched noise (inj≥8 AND ratio>5): 0.2× (tier-2)
+ *
+ * Old thresholds (v26→v2.46, inj≥10/≥20) were chosen as theoretical upper bounds
+ * before injection_count accumulated 2 months of data — live distribution shows
+ * 100% of rows stayed under 10 inject events. The recalibrated gates bite the
+ * moderate-noise tier (first real data band) while still sparing ratio-clean
+ * heavy-use rows (ratio gate is the primary precision signal).
  *
  * Applied as: BM25 × time_decay × TYPE_QUALITY × (0.5 + 0.5·importance) × NOISE_PENALTY
  * Note: multiplicative so ORDER BY relevance ASC (negative scores) still works —
@@ -88,10 +93,10 @@ export function noisePenaltyClause(alias = 'o') {
   const a = alias ? `${alias}.` : '';
   return `(
     CASE
-      WHEN COALESCE(${a}injection_count, 0) >= 20
+      WHEN COALESCE(${a}injection_count, 0) >= 8
         AND COALESCE(${a}injection_count, 0) > COALESCE(${a}access_count, 0) * 5
         THEN 0.2
-      WHEN COALESCE(${a}injection_count, 0) >= 10
+      WHEN COALESCE(${a}injection_count, 0) >= 4
         AND COALESCE(${a}injection_count, 0) > COALESCE(${a}access_count, 0) * 3
         THEN 0.5
       ELSE 1.0

package/tfidf.mjs

@@ -252,6 +252,13 @@ export function rebuildVocabulary(db) {
     for (const [term, entry] of vocab.terms) {
       insertStmt.run(term, entry.index, entry.idf, vocab.version, now);
     }
+    // v2.47 P0-1: drop observation_vectors from earlier vocab versions.
+    // Without this, rebuildVocabulary compounded the stale set on every call
+    // (live DB measured 3282/6429 = 51% stale). vectorSearch filters by
+    // vocab_version at query time, so stale rows were dead storage.
+    try {
+      db.prepare('DELETE FROM observation_vectors WHERE vocab_version != ?').run(vocab.version);
+    } catch { /* table missing on legacy DBs — non-critical */ }
   })();
 
   _vocabCache = vocab;