claude-mem-lite 2.73.2 → 2.75.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.73.2",
+      "version": "2.75.0",
       "source": "./",
       "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.73.2",
+  "version": "2.75.0",
   "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
   "author": {
     "name": "sdsrss"

package/cli.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const CLI_COMMANDS = new Set(['search', 'recent', 'recall', 'get', 'timeline', 'save', 'stats', 'context', 'browse', 'delete', 'update', 'export', 'compress', 'maintain', 'optimize', 'fts-check', 'registry', 'import', 'import-jsonl', 'enrich', 'activity', 'adopt', 'unadopt', 'memdir-audit', 'defer', 'help']);
+const CLI_COMMANDS = new Set(['search', 'recent', 'recall', 'get', 'timeline', 'save', 'stats', 'context', 'browse', 'citation-stats', 'delete', 'update', 'export', 'compress', 'maintain', 'optimize', 'fts-check', 'registry', 'import', 'import-jsonl', 'enrich', 'activity', 'adopt', 'unadopt', 'memdir-audit', 'defer', 'help']);
 const INSTALL_COMMANDS = new Set(['install', 'uninstall', 'status', 'doctor', 'cleanup', 'cleanup-hooks', 'self-update', 'release']);
 
 const cmd = process.argv[2];

package/hook.mjs

@@ -45,7 +45,13 @@ import {
 } from './hook-shared.mjs';
 import { handleLLMEpisode, handleLLMSummary, saveObservation, buildImmediateObservation } from './hook-llm.mjs';
 import { scrubRecord } from './lib/scrub-record.mjs';
-import { extractCitationsFromTranscript, bumpCitationAccess, computeCiteRecall } from './lib/citation-tracker.mjs';
+import {
+  extractCitationsFromTranscript,
+  extractInjectedFromPreToolUse,
+  bumpCitationAccess,
+  computeCiteRecall,
+  applyCitationDecay,
+} from './lib/citation-tracker.mjs';
 import { extractTailAssistantText, extractStructuredSummary } from './lib/summary-extractor.mjs';
 import { searchRelevantMemories, formatMemoryLine } from './hook-memory.mjs';
 import { detectMemOverride } from './lib/mem-override.mjs';
@@ -504,6 +510,12 @@ async function handleStop() {
       // P4: scan transcript for `#NN` observation citations in assistant text
       // and bump access_count for matched rows. Closes the loop on the "cite #NN"
       // contract — before P4 this was a one-way obligation with no feedback.
+      //
+      // CLAUDE_MEM_NO_CITATION_TRACK=1 disables BOTH the P4 access_count bump
+      // AND the v32 citation-decay loop nested below — anything that needs the
+      // transcript scan lives inside this guard. To disable just the decay
+      // loop (keep access_count bumps), use MEM_DISABLE_CITATION_DECAY=1 which
+      // applyCitationDecay checks separately.
       try {
         if (transcriptPath && !process.env.CLAUDE_MEM_NO_CITATION_TRACK) {
           const ids = extractCitationsFromTranscript(transcriptPath);
@@ -512,6 +524,19 @@ async function handleStop() {
             debugLog('DEBUG', 'handleStop', `citations: ${ids.size} ids scanned, ${n} obs bumped`);
           }
 
+          // v32 citation-decay: tighter feedback loop on top of P4. Re-scan
+          // transcript with main-thread filter, extract injected IDs from
+          // pre-tool-recall attachments only, then mutate importance/streak per
+          // applyCitationDecay's contract. Cheap (file still in OS cache).
+          try {
+            const injected = extractInjectedFromPreToolUse(transcriptPath);
+            if (injected.size > 0) {
+              const citedMain = extractCitationsFromTranscript(transcriptPath, { mainOnly: true });
+              const r = applyCitationDecay(db, project, injected, citedMain, sessionId);
+              debugLog('DEBUG', 'handleStop', `citation-decay: touched=${r.touched} promoted=${r.promoted} demoted=${r.demoted}`);
+            }
+          } catch (e) { debugCatch(e, 'handleStop-citation-decay'); }
+
           // Persist cite-recall ratio for the next SessionStart to surface as
           // feedback. We deliberately scan the transcript a second time here
           // (cheap; the file is already in OS cache) rather than threading the

package/lib/citation-tracker.mjs

@@ -22,9 +22,12 @@ const CITATION_RE = /#(\d{1,7})\b/g;
  * cited inside assistant text blocks.
  *
  * @param {string} transcriptPath Path to transcript file (.jsonl)
+ * @param {object} [opts] Options
+ * @param {boolean} [opts.mainOnly=false] If true, skip transcript records where isSidechain === true
  * @returns {Set<number>} unique IDs referenced as `#NN` in assistant text
  */
-export function extractCitationsFromTranscript(transcriptPath) {
+export function extractCitationsFromTranscript(transcriptPath, opts = {}) {
+  const { mainOnly = false } = opts;
   const ids = new Set();
   if (!transcriptPath || !existsSync(transcriptPath)) return ids;
   let raw;
@@ -35,6 +38,11 @@ export function extractCitationsFromTranscript(transcriptPath) {
     try { entry = JSON.parse(line); } catch { continue; }
     // Claude Code transcript: one JSON per line with type='assistant' | 'user' | ...
     if (entry.type !== 'assistant' || !entry.message) continue;
+    // Citation-decay loop scopes citation signal to main-thread text only —
+    // subagent dispatches run their own session context the parent can't
+    // reasonably be held accountable for. Default off preserves the broader
+    // access_count-bump semantics of existing callers (P4 bumpCitationAccess).
+    if (mainOnly && entry.isSidechain === true) continue;
     const content = entry.message.content;
     if (!Array.isArray(content)) continue;
     for (const block of content) {
@@ -142,3 +150,137 @@ export function bumpCitationAccess(db, ids, project) {
   }
   return n;
 }
+
+// Matches a pre-tool-recall lesson line: `  #NN [type] body...`. Bounded type
+// list mirrors observations.type CHECK + the events table's allowed event_type
+// values pre-tool-recall.js can surface.
+const INJECTED_RE = /#(\d{1,7})\s+\[(bugfix|decision|change|discovery|feature|refactor|lesson)\]/g;
+
+/**
+ * Extract observation IDs injected by pre-tool-recall hook in this transcript.
+ *
+ * Tighter than `computeCiteRecall`'s over-inclusive "any #NN in non-assistant
+ * text" — only counts IDs the agent actually saw from us, not user-pasted
+ * references or unrelated #NN tokens in tool output.
+ *
+ * @param {string|null|undefined} transcriptPath
+ * @returns {Set<number>} unique injected IDs (empty set on missing path/file)
+ */
+export function extractInjectedFromPreToolUse(transcriptPath) {
+  const ids = new Set();
+  if (!transcriptPath || !existsSync(transcriptPath)) return ids;
+  let raw;
+  try { raw = readFileSync(transcriptPath, 'utf8'); } catch { return ids; }
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    let entry;
+    try { entry = JSON.parse(line); } catch { continue; }
+    if (entry.type !== 'attachment') continue;
+    const att = entry.attachment;
+    if (!att || att.type !== 'hook_success') continue;
+    if (!(att.command || '').includes('pre-tool-recall')) continue;
+    const stdout = att.stdout || '';
+    if (!stdout) continue;
+    // stdout is JSON wrapping additionalContext OR raw text (legacy);
+    // try JSON first and fall back to raw.
+    let text = stdout;
+    try {
+      const parsed = JSON.parse(stdout);
+      text = parsed?.hookSpecificOutput?.additionalContext || stdout;
+    } catch {}
+    INJECTED_RE.lastIndex = 0;
+    let m;
+    while ((m = INJECTED_RE.exec(text))) {
+      const id = Number(m[1]);
+      if (Number.isInteger(id) && id > 0 && id < 1e7) ids.add(id);
+    }
+  }
+  return ids;
+}
+
+const IMPORTANCE_CAP = 3;
+const IMPORTANCE_FLOOR = 0;
+const UNCITED_STREAK_THRESHOLD = 3;
+
+/**
+ * Apply the citation-feedback loop for one session: for each injected obs id,
+ * decide cited vs uncited and mutate importance/streak/cited_count per spec.
+ *
+ * - cited: importance += 1 (cap 3), cited_count += 1, streak = 0.
+ * - uncited: streak += 1; if it reaches 3, importance -= 1 (floor 0), streak = 0.
+ * - per-(session, obs) idempotent via last_decided_session_id; re-running for
+ *   the same session is a no-op (Stop hook may fire more than once).
+ * - cross-project IDs are silently ignored by the WHERE clause.
+ * - MEM_DISABLE_CITATION_DECAY=1 disables all writes; returns zeros.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} project
+ * @param {Set<number>|Iterable<number>} injectedIds
+ * @param {Set<number>|Iterable<number>} citedIds
+ * @param {string} sessionId — memory_session_id of the session being resolved
+ * @returns {{promoted: number, demoted: number, touched: number}}
+ */
+export function applyCitationDecay(db, project, injectedIds, citedIds, sessionId) {
+  const empty = { promoted: 0, demoted: 0, touched: 0 };
+  if (process.env.MEM_DISABLE_CITATION_DECAY === '1') return empty;
+  if (!db || !project || !sessionId) return empty;
+  const injected = injectedIds instanceof Set ? injectedIds : new Set(injectedIds || []);
+  if (injected.size === 0) return empty;
+  const cited = citedIds instanceof Set ? citedIds : new Set(citedIds || []);
+
+  const selectStmt = db.prepare(
+    'SELECT id, importance, uncited_streak, last_decided_session_id FROM observations WHERE id = ? AND project = ?'
+  );
+  // decay_seen_count (v34) bumps on every resolution branch — gives
+  // citation-stats a denominator that's same-source as cited_count, so the
+  // ratio actually means "cite-rate" instead of mixing decay + UserPromptSubmit.
+  const updatePromote = db.prepare(`
+    UPDATE observations
+       SET importance = MIN(?, importance + 1),
+           cited_count = cited_count + 1,
+           uncited_streak = 0,
+           last_decided_session_id = ?,
+           decay_seen_count = decay_seen_count + 1
+     WHERE id = ?
+  `);
+  const updateStreakOnly = db.prepare(`
+    UPDATE observations
+       SET uncited_streak = uncited_streak + 1,
+           last_decided_session_id = ?,
+           decay_seen_count = decay_seen_count + 1
+     WHERE id = ?
+  `);
+  const updateDemote = db.prepare(`
+    UPDATE observations
+       SET importance = MAX(?, importance - 1),
+           uncited_streak = 0,
+           last_decided_session_id = ?,
+           demoted_at = ?,
+           decay_seen_count = decay_seen_count + 1
+     WHERE id = ?
+  `);
+
+  let promoted = 0, demoted = 0, touched = 0;
+  const txn = db.transaction(() => {
+    for (const id of injected) {
+      const row = selectStmt.get(id, project);
+      if (!row) continue; // cross-project or deleted
+      if (row.last_decided_session_id === sessionId) continue; // idempotent skip
+      touched++;
+      if (cited.has(id)) {
+        updatePromote.run(IMPORTANCE_CAP, sessionId, id);
+        promoted++;
+      } else {
+        const nextStreak = (row.uncited_streak || 0) + 1;
+        if (nextStreak >= UNCITED_STREAK_THRESHOLD) {
+          updateDemote.run(IMPORTANCE_FLOOR, sessionId, Date.now(), id);
+          demoted++;
+        } else {
+          updateStreakOnly.run(sessionId, id);
+        }
+      }
+    }
+  });
+  try { txn(); } catch (e) { debugCatch(e, 'applyCitationDecay-txn'); return empty; }
+  return { promoted, demoted, touched };
+}

package/mem-cli.mjs

@@ -2358,6 +2358,96 @@ function cmdMemdirAudit(args) {
   if (nonCompliant > 0) process.exitCode = 1;
 }
 
+/**
+ * `citation-stats` — visualize the citation-decay feedback loop:
+ * per-project cite rate + active decay queue + recently promoted.
+ * Read-only over observations.
+ *
+ * Flags:
+ *   --json      machine-readable output
+ *   --days N    project cite-rate window (default 7)
+ */
+function cmdCitationStats(db, args) {
+  const { flags } = parseArgs(args);
+  const json = flags.json === true || flags.json === 'true';
+  const days = parseIntFlag(flags.days, { name: '--days', defaultValue: 7, max: 365 });
+
+  const cutoff = Date.now() - days * 86400 * 1000;
+  const perProject = db.prepare(`
+    SELECT project,
+           COALESCE(SUM(cited_count), 0) AS cited,
+           COALESCE(SUM(decay_seen_count), 0) AS resolved,
+           SUM(CASE WHEN uncited_streak >= 2 THEN 1 ELSE 0 END) AS at_risk
+      FROM observations
+     WHERE created_at_epoch >= ?
+       AND COALESCE(compressed_into, 0) = 0
+       AND superseded_at IS NULL
+  GROUP BY project
+  ORDER BY resolved DESC
+  `).all(cutoff);
+
+  const decayQueue = db.prepare(`
+    SELECT id, project, type, title, importance, uncited_streak, cited_count
+      FROM observations
+     WHERE uncited_streak >= 2
+       AND COALESCE(compressed_into, 0) = 0
+       AND superseded_at IS NULL
+  ORDER BY uncited_streak DESC, importance ASC
+     LIMIT 20
+  `).all();
+
+  const promoted = db.prepare(`
+    SELECT id, project, type, title, importance, cited_count
+      FROM observations
+     WHERE importance >= 3 AND cited_count >= 1
+       AND COALESCE(compressed_into, 0) = 0
+       AND superseded_at IS NULL
+  ORDER BY cited_count DESC
+     LIMIT 10
+  `).all();
+
+  const demoted = db.prepare(`
+    SELECT id, project, type, title, importance, demoted_at
+      FROM observations
+     WHERE demoted_at IS NOT NULL
+       AND demoted_at >= ?
+       AND COALESCE(compressed_into, 0) = 0
+       AND superseded_at IS NULL
+  ORDER BY demoted_at DESC
+     LIMIT 10
+  `).all(cutoff);
+
+  if (json) {
+    out(JSON.stringify({ window_days: days, per_project: perProject, decay_queue: decayQueue, promoted, demoted }, null, 2));
+    return;
+  }
+
+  out(`Cite rate by project (last ${days}d, cited / decay-resolutions):`);
+  for (const r of perProject) {
+    const rate = r.resolved > 0 ? (r.cited * 100 / r.resolved).toFixed(1) + '%' : '—';
+    out(`  ${r.project.padEnd(34)} ${String(rate).padStart(6)}   cited:${r.cited}/${r.resolved}   at_risk:${r.at_risk}`);
+  }
+  out('');
+  out('Active decay queue (uncited_streak >= 2, next miss → demote):');
+  if (decayQueue.length === 0) out('  (none)');
+  for (const r of decayQueue) {
+    out(`  #${r.id} [${r.type}] ${(r.title || '').slice(0, 60)}   imp=${r.importance} streak=${r.uncited_streak}`);
+  }
+  out('');
+  out('Recently promoted (importance=3, cited_count >= 1):');
+  if (promoted.length === 0) out('  (none)');
+  for (const r of promoted) {
+    out(`  #${r.id} [${r.type}] ${(r.title || '').slice(0, 60)}   cited ${r.cited_count}x`);
+  }
+  out('');
+  out(`Recently demoted (last ${days}d, importance ↓):`);
+  if (demoted.length === 0) out('  (none)');
+  for (const r of demoted) {
+    const ago = Math.round((Date.now() - r.demoted_at) / 86400000);
+    out(`  #${r.id} [${r.type}] ${(r.title || '').slice(0, 60)}   imp=${r.importance}   ${ago}d ago`);
+  }
+}
+
 // ─── Help ────────────────────────────────────────────────────────────────────
 
 function cmdHelp() {
@@ -2498,6 +2588,10 @@ Commands:
                           totals:{working,active,archive,grand_total},
                           tiers:{working:{count,results:[…]}, …}}
 
+  citation-stats        Citation-decay feedback loop telemetry
+    --days N            Cite-rate window in days (default 7)
+    --json              Output as JSON: {window_days,per_project:[],decay_queue:[],promoted:[]}
+
   registry <action>     Manage tool resource registry
     list                List resources [--type skill|agent] [--limit N] (default 20)
     stats               Registry statistics
@@ -2867,7 +2961,7 @@ export async function run(argv) {
   // text-parsing callers keep working — the note lives in stderr for scripts to
   // detect the gap.
   const JSON_SUPPORTED_CMDS = new Set([
-    'search', 'context', 'recent', 'recall', 'timeline', 'stats', 'browse', 'export',
+    'search', 'context', 'recent', 'recall', 'timeline', 'stats', 'browse', 'export', 'citation-stats',
   ]);
   // `doctor --benchmark` already emits JSON on its own — don't print the misleading
   // "doctor outputs text" note for that subpath. Without --benchmark, doctor is text
@@ -2896,6 +2990,7 @@ export async function run(argv) {
       case 'stats':     await cmdStats(db, cmdArgs); break;
       case 'context':   cmdContext(db, cmdArgs); break;
       case 'browse':    cmdBrowse(db, cmdArgs); break;
+      case 'citation-stats': cmdCitationStats(db, cmdArgs); break;
       case 'registry':  cmdRegistry(db, cmdArgs); break;
       case 'import':    await cmdImport(cmdArgs); break;
       case 'import-jsonl': await cmdImportJsonl(db, cmdArgs); break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.73.2",
+  "version": "2.75.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "packageManager": "npm@10.9.2",

package/schema.mjs

@@ -45,7 +45,28 @@ export const REGISTRY_DB_PATH = join(DB_DIR, 'resource-registry.db');
 // items rank HIGHER as tech debt accumulates), different lifecycle (mutable
 // status open→done|dropped vs immutable obs). Closure tied to obs via
 // closed_by_obs_id FK with ON DELETE SET NULL (audit trail preserved).
-export const CURRENT_SCHEMA_VERSION = 31;
+// v32 (v2.73.2): citation-decay columns on observations — uncited_streak,
+// cited_count, last_decided_session_id. Stop hook resolves injected obs as
+// cited|uncited; 3 consecutive uncited → importance -1 (floor 0); 1 cited → +1
+// (cap 3). last_decided_session_id makes Stop idempotent across multi-fire.
+export const CURRENT_SCHEMA_VERSION = 34;
+
+// Sentinel column for the LATEST migration set. The fast-path uses this to
+// self-heal half-migrated DBs — schema_version bumped but column ALTERs rolled
+// back (observed once in dev during v2.74.0). Update both the column AND
+// (if needed) the table when adding a new migration batch.
+const LATEST_MIGRATION_COLUMN = { table: 'observations', column: 'decay_seen_count' };
+
+function hasLatestMigrationColumn(db) {
+  try {
+    const row = db.prepare(
+      `SELECT 1 AS present FROM pragma_table_info(?) WHERE name = ?`
+    ).get(LATEST_MIGRATION_COLUMN.table, LATEST_MIGRATION_COLUMN.column);
+    return Boolean(row);
+  } catch {
+    return false; // table itself missing → caller falls through to CORE_SCHEMA
+  }
+}
 
 const CORE_SCHEMA = `
   CREATE TABLE IF NOT EXISTS sdk_sessions (
@@ -151,6 +172,27 @@ const MIGRATIONS = [
   // injection_count + low access_count = low-signal, deprioritize.
   'ALTER TABLE observations ADD COLUMN injection_count INTEGER NOT NULL DEFAULT 0',
   'ALTER TABLE observations ADD COLUMN last_injected_at INTEGER DEFAULT NULL',
+  // v32 (citation-decay): per-obs feedback loop for pre-tool-recall injection
+  // pool. Stop hook resolves each session's injected IDs as cited|uncited.
+  // 3 consecutive uncited sessions → importance -1 (floor 0). 1 cited session →
+  // importance +1 (cap 3). last_decided_session_id makes Stop idempotent across
+  // multi-fire scenarios (Claude may fire Stop more than once per session).
+  'ALTER TABLE observations ADD COLUMN uncited_streak INTEGER NOT NULL DEFAULT 0',
+  'ALTER TABLE observations ADD COLUMN cited_count INTEGER NOT NULL DEFAULT 0',
+  'ALTER TABLE observations ADD COLUMN last_decided_session_id TEXT DEFAULT NULL',
+  // v33 (citation-decay telemetry): timestamp of the most recent demote event.
+  // Powers `claude-mem-lite citation-stats`'s "Recently demoted" section.
+  // Set in applyCitationDecay's demote branch when streak hits threshold.
+  // Single-shot (only the latest demote is preserved); use a decay_log table
+  // if historical trend is ever needed.
+  'ALTER TABLE observations ADD COLUMN demoted_at INTEGER DEFAULT NULL',
+  // v34 (citation-decay denominator fix): per-obs counter of decay-loop
+  // resolutions (cited + uncited paths). Used by citation-stats as the
+  // denominator for "cite rate" — bumped only by applyCitationDecay, so it
+  // doesn't get polluted by UserPromptSubmit / hook-memory injections that
+  // share the unrelated injection_count column. Same-source numerator
+  // (cited_count) + same-source denominator = meaningful ratio.
+  'ALTER TABLE observations ADD COLUMN decay_seen_count INTEGER NOT NULL DEFAULT 0',
 ];
 
 /**
@@ -167,7 +209,10 @@ export function initSchema(db) {
   try {
     const row = db.prepare('SELECT version FROM schema_version LIMIT 1').get();
     if (row && typeof row.version === 'number') {
-      if (row.version === CURRENT_SCHEMA_VERSION) return db;
+      // Self-heal: version-row says CURRENT but latest migration column may be
+      // absent (rolled back / never applied). Fall through to migration apply
+      // when the sentinel is missing — duplicates are caught in the loop.
+      if (row.version === CURRENT_SCHEMA_VERSION && hasLatestMigrationColumn(db)) return db;
       if (row.version > CURRENT_SCHEMA_VERSION) {
         throw new Error(
           `DB schema is v${row.version} but this claude-mem-lite binary supports up to v${CURRENT_SCHEMA_VERSION}. ` +
@@ -190,7 +235,7 @@ export function initSchema(db) {
   db.exec('BEGIN IMMEDIATE');
   try {
     const underlock = db.prepare('SELECT version FROM schema_version LIMIT 1').get();
-    if (underlock && underlock.version === CURRENT_SCHEMA_VERSION) {
+    if (underlock && underlock.version === CURRENT_SCHEMA_VERSION && hasLatestMigrationColumn(db)) {
       db.exec('COMMIT');
       return db;
     }