claude-mem-lite 2.36.0 → 2.37.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.36.0",
+      "version": "2.37.0",
       "source": "./",
       "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.36.0",
+  "version": "2.37.0",
   "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
   "author": {
     "name": "sdsrss"

package/hook-memory.mjs

@@ -1,7 +1,7 @@
 // claude-mem-lite — Semantic Memory Injection
 // Search past observations for relevant memories to inject as context at user-prompt time.
 
-import { sanitizeFtsQuery, relaxFtsQueryToOr, debugCatch, OBS_BM25, notLowSignalTitleClause } from './utils.mjs';
+import { sanitizeFtsQuery, relaxFtsQueryToOr, debugCatch, OBS_BM25, notLowSignalTitleClause, noisePenaltyClause } from './utils.mjs';
 
 const MAX_MEMORY_INJECTIONS = 3;
 const MEMORY_LOOKBACK_MS = 60 * 86400000; // 60 days
@@ -42,9 +42,14 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
     // R1: notLowSignalTitleClause() excludes hook-llm fallback titles
     // ("Modified X", "Worked on X", "Reviewed N files:", raw error logs, etc.)
     // that almost never get referenced (3.3% access rate) but compete for BM25 rank.
+    // v26 P0: noise_penalty is multiplied AFTER sort-BM25 so the column used
+    // for ORDER BY stays the penalty-adjusted `relevance` applied downstream
+    // in JS (scored.sort). SELECT exposes both raw BM25 (for sort) and the
+    // penalty factor (for the final JS score).
     const selectStmt = db.prepare(`
       SELECT o.id, o.type, o.title, o.importance, o.lesson_learned, o.project,
-             ${OBS_BM25} as relevance
+             ${OBS_BM25} as relevance,
+             ${noisePenaltyClause('o')} as noise_penalty
       FROM observations_fts
       JOIN observations o ON o.id = observations_fts.rowid
       WHERE observations_fts MATCH ?
@@ -80,7 +85,8 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
     try {
       const crossStmt = db.prepare(`
         SELECT o.id, o.type, o.title, o.importance, o.lesson_learned, o.project,
-               ${OBS_BM25} as relevance
+               ${OBS_BM25} as relevance,
+               ${noisePenaltyClause('o')} as noise_penalty
         FROM observations_fts
         JOIN observations o ON o.id = observations_fts.rowid
         WHERE observations_fts MATCH ?
@@ -105,12 +111,14 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
 
     // Merge and score: same-project full weight, cross-project 0.7x
     // OR-fallback results get 0.4x penalty — they matched individual words, not the full intent
+    // v26 P0: noise_penalty (from SQL) shrinks high-inject/low-cite rows.
     const allRows = [...rows.map(r => ({ ...r, _or: usedOrFallback })), ...crossRows.map(r => ({ ...r, _or: crossUsedOr }))];
     const scored = allRows
       .filter(r => !excludeSet.has(r.id))
       .map(r => {
         const crossProjectPenalty = r.project === project ? 1.0 : 0.7;
         const orFallbackPenalty = r._or ? 0.4 : 1.0;
+        const noisePenalty = typeof r.noise_penalty === 'number' ? r.noise_penalty : 1.0;
         return {
           ...r,
           score: Math.abs(r.relevance)
@@ -118,7 +126,8 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
             * (r.lesson_learned ? 1.5 : 1.0)
             * (r.importance >= 2 ? 1.0 : 0.6)
             * crossProjectPenalty
-            * orFallbackPenalty,
+            * orFallbackPenalty
+            * noisePenalty,
         };
       })
       .sort((a, b) => b.score - a.score);
@@ -133,12 +142,19 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
     const aboveThreshold = scored.filter(r => r.score >= threshold);
     if (aboveThreshold.length === 0) return [];
 
-    // Update access_count for injected memories
+    // v26 P0: bump injection_count (NOT access_count) for injected rows.
+    // Before v26 this was bumping access_count, which conflated auto-injection
+    // with real cites/recalls/opens — polluting the noise-ratio signal the
+    // penalty clause now depends on. access_count is reserved for explicit
+    // access (cmdRecall/cmdGet/cmdTimeline/pre-tool-recall/citation-tracker).
+    // Per-row try/catch for FTS trigger safety (project_non_obvious.md).
     const result = aboveThreshold.slice(0, MAX_MEMORY_INJECTIONS);
     const now = Date.now();
-    const updateStmt = db.prepare('UPDATE observations SET access_count = COALESCE(access_count, 0) + 1, last_accessed_at = ? WHERE id = ?');
+    const bumpStmt = db.prepare(
+      'UPDATE observations SET injection_count = COALESCE(injection_count, 0) + 1, last_injected_at = ? WHERE id = ?'
+    );
     for (const r of result) {
-      updateStmt.run(now, r.id);
+      try { bumpStmt.run(now, r.id); } catch {}
     }
 
     return result;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.36.0",
+  "version": "2.37.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "engines": {

package/schema.mjs

@@ -13,7 +13,7 @@ export const DB_PATH = join(DB_DIR, 'claude-mem-lite.db');
 export const REGISTRY_DB_PATH = join(DB_DIR, 'resource-registry.db');
 
 // Increment when schema changes (tables, columns, indexes, FTS, migrations)
-export const CURRENT_SCHEMA_VERSION = 25;
+export const CURRENT_SCHEMA_VERSION = 26;
 
 const CORE_SCHEMA = `
   CREATE TABLE IF NOT EXISTS sdk_sessions (
@@ -112,6 +112,13 @@ const MIGRATIONS = [
   'ALTER TABLE observations ADD COLUMN superseded_by INTEGER DEFAULT NULL',
   'ALTER TABLE observations ADD COLUMN last_accessed_at INTEGER DEFAULT NULL',
   'ALTER TABLE observations ADD COLUMN optimized_at INTEGER DEFAULT NULL',
+  // v26 (P0 injection-noise): per-obs injection tracking for noise-ratio
+  // penalty. injection_count bumps only on UserPromptSubmit / hook-memory
+  // auto-injection (not on explicit recall/get/timeline — those keep bumping
+  // access_count). Pair with access_count to compute noise ratio: high
+  // injection_count + low access_count = low-signal, deprioritize.
+  'ALTER TABLE observations ADD COLUMN injection_count INTEGER NOT NULL DEFAULT 0',
+  'ALTER TABLE observations ADD COLUMN last_injected_at INTEGER DEFAULT NULL',
 ];
 
 /**

package/scoring-sql.mjs

@@ -61,6 +61,44 @@ export const TYPE_QUALITY_CASE = `(
   END
 )`;
 
+/**
+ * Noise-ratio penalty: deprioritizes observations that get auto-injected often
+ * but rarely "used" (cited via Stop-hook citation tracker, or explicitly
+ * recalled/opened via pre-tool-recall / cmdRecall / cmdGet / cmdTimeline).
+ *
+ * Signal sources:
+ *   - injection_count: bumped ONLY on UserPromptSubmit / hook-memory auto-inject
+ *   - access_count: bumped on citation (c039352 P4), explicit recall, get, timeline
+ *
+ * Empirical thresholds (see docs/p0-injection-noise-baseline.txt, 53 transcripts):
+ *   • High-noise legitimate use (#5597 29/10=2.9x): kept at 1.0× (below tier-1)
+ *   • Moderate noise (#4352 44/9=4.89x): drops to 0.5× (tier-1 hit)
+ *   • Pure noise (#4046 14/0=inf): drops to 0.5× (tier-1; count≥10 gate protects
+ *     cold-start obs with legitimately no cites yet)
+ *   • Entrenched noise (≥20 inject, ≥5× ratio): drops to 0.2× (tier-2)
+ *
+ * Applied as: BM25 × time_decay × TYPE_QUALITY × (0.5 + 0.5·importance) × NOISE_PENALTY
+ * Note: multiplicative so ORDER BY relevance ASC (negative scores) still works —
+ * penalty shrinks magnitude, making the row less preferable.
+ *
+ * @param {string} [alias='o'] Table alias for the observations row.
+ * @returns {string} SQL CASE expression (already parenthesized).
+ */
+export function noisePenaltyClause(alias = 'o') {
+  const a = alias ? `${alias}.` : '';
+  return `(
+    CASE
+      WHEN COALESCE(${a}injection_count, 0) >= 20
+        AND COALESCE(${a}injection_count, 0) > COALESCE(${a}access_count, 0) * 5
+        THEN 0.2
+      WHEN COALESCE(${a}injection_count, 0) >= 10
+        AND COALESCE(${a}injection_count, 0) > COALESCE(${a}access_count, 0) * 3
+        THEN 0.5
+      ELSE 1.0
+    END
+  )`;
+}
+
 /**
  * SQL WHERE clause fragment excluding LOW_SIGNAL degraded titles — the fallback
  * titles hook-llm.mjs writes when Haiku summarization is unavailable or skipped

package/scripts/user-prompt-search.js

@@ -4,7 +4,7 @@
 // Lightweight: only imports schema.mjs and utils.mjs, no MCP SDK
 
 import { ensureDb, DB_DIR, REGISTRY_DB_PATH } from '../schema.mjs';
-import { sanitizeFtsQuery, relaxFtsQueryToOr, truncate, typeIcon, inferProject, OBS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, notLowSignalTitleClause } from '../utils.mjs';
+import { sanitizeFtsQuery, relaxFtsQueryToOr, truncate, typeIcon, inferProject, OBS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, notLowSignalTitleClause, noisePenaltyClause } from '../utils.mjs';
 import { writeFileSync, readFileSync, existsSync, renameSync } from 'fs';
 import { join } from 'path';
 import Database from 'better-sqlite3';
@@ -87,12 +87,16 @@ function searchByFts(db, queryText, project, limit, typeFilter) {
   const now = Date.now();
   // R1: notLowSignalTitleClause() excludes hook-llm degraded titles
   // ("Modified X", "Worked on X", "Reviewed N files:", raw error logs).
+  // v26 P0: noise penalty shrinks relevance magnitude for obs with high
+  // inject:access ratio (auto-injected often, never cited/opened). See
+  // docs/p0-injection-noise-baseline.txt.
   const sql = `
     SELECT o.id, o.type, o.title, o.lesson_learned,
            ${OBS_BM25}
              * (1.0 + EXP(-0.693 * (? - o.created_at_epoch) / ${TYPE_DECAY_CASE}))
              * ${TYPE_QUALITY_CASE}
-             * (0.5 + 0.5 * COALESCE(o.importance, 1)) as relevance
+             * (0.5 + 0.5 * COALESCE(o.importance, 1))
+             * ${noisePenaltyClause('o')} as relevance
     FROM observations_fts
     JOIN observations o ON o.id = observations_fts.rowid
     WHERE observations_fts MATCH ?
@@ -460,6 +464,22 @@ async function main() {
           count: prevCount + 1,
         }));
       } catch {}
+      // v26 P0: bump injection_count for obs-based emits only (prompt-corpus
+      // rows have "P<id>" string IDs; skip those — they live in user_prompts).
+      // Per-row try/catch: observations_au trigger reinserts FTS on any UPDATE
+      // (project_non_obvious.md); an FTS corruption on one row must not abort
+      // counter bumps for other rows.
+      if (rows.length > 0) {
+        try {
+          const now = Date.now();
+          const bumpStmt = db.prepare(
+            'UPDATE observations SET injection_count = COALESCE(injection_count, 0) + 1, last_injected_at = ? WHERE id = ?'
+          );
+          for (const r of rows) {
+            try { bumpStmt.run(now, r.id); } catch {}
+          }
+        } catch {}
+      }
     }
 
     // ─── L1: Registry skill pointer (T4 v2.31) ──────────────────────────

package/utils.mjs

@@ -9,7 +9,7 @@ import { buildLowSignalRegex } from './lib/low-signal-patterns.mjs';
 // ─── Re-exports from extracted modules ──────────────────────────────────────
 // Backward compatibility: all consumers import from utils.mjs
 
-export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause } from './scoring-sql.mjs';
+export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
 export { cjkBigrams, extractCjkSynonymTokens, extractCjkKeywords, extractCjkLikePatterns, SYNONYM_MAP, expandToken, sanitizeFtsQuery, relaxFtsQueryToOr, FTS_STOP_WORDS, CJK_COMPOUNDS } from './nlp.mjs';
 export { resolveProject, _resetProjectCache } from './project-utils.mjs';
 export { scrubSecrets, SECRET_PATTERNS } from './secret-scrub.mjs';