claude-mem-lite 2.75.0 → 2.77.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.75.0",
+      "version": "2.77.0",
       "source": "./",
       "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.75.0",
+  "version": "2.77.0",
   "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
   "author": {
     "name": "sdsrss"

package/hook.mjs

@@ -47,7 +47,7 @@ import { handleLLMEpisode, handleLLMSummary, saveObservation, buildImmediateObse
 import { scrubRecord } from './lib/scrub-record.mjs';
 import {
   extractCitationsFromTranscript,
-  extractInjectedFromPreToolUse,
+  extractAllInjected,
   bumpCitationAccess,
   computeCiteRecall,
   applyCitationDecay,
@@ -525,11 +525,17 @@ async function handleStop() {
           }
 
           // v32 citation-decay: tighter feedback loop on top of P4. Re-scan
-          // transcript with main-thread filter, extract injected IDs from
-          // pre-tool-recall attachments only, then mutate importance/streak per
-          // applyCitationDecay's contract. Cheap (file still in OS cache).
+          // transcript with main-thread filter, extract injected IDs from BOTH
+          // surfaces (PTR + UserPromptSubmit <memory-context>) via extractAllInjected,
+          // then mutate importance/streak per applyCitationDecay's contract.
+          // Cheap (file still in OS cache).
+          //
+          // v34.x: pre-v34 this only saw pre-tool-recall injections, leaving the
+          // UPS surface (highest-volume — all decision-type FTS hits) starved.
+          // Union closed by extractAllInjected — one integration point so the
+          // contract test in tests/citation-tracker-userprompt.test.mjs covers it.
           try {
-            const injected = extractInjectedFromPreToolUse(transcriptPath);
+            const injected = extractAllInjected(transcriptPath);
             if (injected.size > 0) {
               const citedMain = extractCitationsFromTranscript(transcriptPath, { mainOnly: true });
               const r = applyCitationDecay(db, project, injected, citedMain, sessionId);

package/lib/citation-tracker.mjs

@@ -198,6 +198,74 @@ export function extractInjectedFromPreToolUse(transcriptPath) {
   return ids;
 }
 
+// v34.x: UserPromptSubmit injection extractor. hook.mjs handleUserPrompt emits
+// formatMemoryLine `- [type] title | Lesson: X (#NN)[ [verify-before-use]]`,
+// which INJECTED_RE (anchored on `#NN [type]`) never matched — leaving the
+// highest-volume injection surface invisible to applyCitationDecay. The two
+// extractors are disjoint by design: PTR has `[type]` AFTER `#NN`, UPS has
+// `(#NN)` at end-of-line.
+//
+// Line-scan with `- [` prefix gate so a lesson body containing a back-reference
+// like "see (#999)" doesn't pollute the injected set (would streak-uncite an
+// obs we never actually displayed as a top-level entry).
+const UPS_LINE_PREFIX = '- [';
+const UPS_ID_RE = /\(#(\d{1,7})\)/g;
+const UPS_COMMAND_SUFFIX = 'hook.mjs user-prompt';
+
+/**
+ * Extract observation IDs injected by the UserPromptSubmit `<memory-context>`
+ * block (hook.mjs handleUserPrompt). Disjoint from pre-tool-recall extraction —
+ * the Stop handler unions both via extractAllInjected.
+ *
+ * @param {string|null|undefined} transcriptPath
+ * @returns {Set<number>}
+ */
+export function extractInjectedFromUserPromptSubmit(transcriptPath) {
+  const ids = new Set();
+  if (!transcriptPath || !existsSync(transcriptPath)) return ids;
+  let raw;
+  try { raw = readFileSync(transcriptPath, 'utf8'); } catch { return ids; }
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    let entry;
+    try { entry = JSON.parse(line); } catch { continue; }
+    if (entry.type !== 'attachment') continue;
+    const att = entry.attachment;
+    if (!att || att.type !== 'hook_success') continue;
+    // Suffix match — survives plugin-cache vs symlinked-install path differences.
+    if (!(att.command || '').includes(UPS_COMMAND_SUFFIX)) continue;
+    const stdout = att.stdout || '';
+    if (!stdout.includes('<memory-context')) continue;
+    for (const memLine of stdout.split('\n')) {
+      if (!memLine.startsWith(UPS_LINE_PREFIX)) continue;
+      // Take the LAST (#NN) on the line — formatMemoryLine puts the obs id
+      // in trailing parens, possibly followed by ` [verify-before-use]`. Any
+      // earlier (#NN) refs are inside title/lesson text (per the test that
+      // pins "see (#999)" → NOT extracted).
+      const matches = [...memLine.matchAll(UPS_ID_RE)];
+      if (matches.length === 0) continue;
+      const id = Number(matches[matches.length - 1][1]);
+      if (Number.isInteger(id) && id > 0 && id < 1e7) ids.add(id);
+    }
+  }
+  return ids;
+}
+
+/**
+ * Union of pre-tool-recall + UserPromptSubmit injection IDs for a transcript.
+ * Single integration point the Stop handler calls — keeps hook.mjs's wiring
+ * a one-liner and gives the contract test something to assert against.
+ *
+ * @param {string|null|undefined} transcriptPath
+ * @returns {Set<number>}
+ */
+export function extractAllInjected(transcriptPath) {
+  return new Set([
+    ...extractInjectedFromPreToolUse(transcriptPath),
+    ...extractInjectedFromUserPromptSubmit(transcriptPath),
+  ]);
+}
+
 const IMPORTANCE_CAP = 3;
 const IMPORTANCE_FLOOR = 0;
 const UNCITED_STREAK_THRESHOLD = 3;

package/lib/hook-telemetry.mjs

@@ -0,0 +1,123 @@
+// lib/hook-telemetry.mjs — unsampled hook-error log.
+//
+// Distinct from lib/err-sampler.mjs: that one writes 1% of swallowed debugCatch
+// errors into ${dbDir}/errors/ for *production diagnostics* — high cardinality,
+// must be cheap. This one writes 100% of hook script failures into
+// ${runtimeDir}/hook-errors/ for *self-observation* — low cardinality (hooks
+// fail rarely), every event matters because it's the only window into
+// PreToolUse / Skill-bridge / UPS failure modes (DB corruption, schema drift,
+// upstream field rename). Both follow the same JSONL daily-shard layout to
+// keep tooling consistent; the directory split is the contract that says
+// "no sampling here, this is full fidelity".
+//
+// Hook scripts catch *all* errors and exit 0 (must never block Edit/Write).
+// Calling this from inside each catch turns the silent path into a recorded
+// path without changing behavior. All failures inside the recorder itself
+// are swallowed — telemetry must never crash the host hook.
+//
+// Retention: 14 days. GC is lazy on append (cheap stat+unlink loop, only
+// reads the dir, no parse).
+
+import { appendFileSync, mkdirSync, existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs';
+import { join } from 'path';
+
+const DAY_MS = 86400000;
+const RETENTION_MS = 14 * DAY_MS;
+const HOOK_ERRORS_SUBDIR = 'hook-errors';
+
+function today() {
+  return new Date(Date.now()).toISOString().slice(0, 10);
+}
+
+function hookErrorsDir(runtimeDir) {
+  return join(runtimeDir, HOOK_ERRORS_SUBDIR);
+}
+
+// Lazy GC: scan the dir, unlink shards older than RETENTION_MS. Cheap because
+// it never opens files — just statSync + unlinkSync. Called from recordHookError
+// after a successful write, so it amortizes over the failure cadence.
+function pruneOldShards(dir) {
+  let entries;
+  try { entries = readdirSync(dir); } catch { return; }
+  const cutoff = Date.now() - RETENTION_MS;
+  for (const f of entries) {
+    if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)) continue;
+    const full = join(dir, f);
+    try {
+      const st = statSync(full);
+      if (st.mtimeMs < cutoff) unlinkSync(full);
+    } catch { /* gone or unreadable — skip */ }
+  }
+}
+
+/**
+ * Record one hook-script failure to the unsampled JSONL log.
+ *
+ * @param {string}  scope       Short label naming the hook script + failure point.
+ *                              Convention: '<script>:<step>' e.g. 'pre-recall:db-open',
+ *                              'skill-bridge:registry-query'. Truncated to 80 chars.
+ * @param {unknown} err         Thrown value (Error, string, undefined — all ok).
+ * @param {string}  runtimeDir  Absolute path to ${CLAUDE_MEM_DIR}/runtime/. Caller
+ *                              must resolve this — module avoids importing
+ *                              hook-shared.mjs so it stays usable from the
+ *                              lightweight standalone scripts.
+ * @param {object}  [ctx]       Optional small JSON-safe context (filePath, toolName,
+ *                              sessionId fragment). Stringified + truncated to 240
+ *                              chars to keep shard lines bounded.
+ */
+export function recordHookError(scope, err, runtimeDir, ctx) {
+  try {
+    if (!runtimeDir || typeof runtimeDir !== 'string') return;
+    const dir = hookErrorsDir(runtimeDir);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 });
+
+    const line = JSON.stringify({
+      ts: new Date().toISOString(),
+      scope: String(scope || '').slice(0, 80),
+      msg: String(err?.message ?? err ?? '').slice(0, 500),
+      stack: typeof err?.stack === 'string' ? err.stack.split('\n').slice(0, 6).join('\n') : undefined,
+      ctx: ctx === undefined ? undefined : JSON.stringify(ctx).slice(0, 240),
+    }) + '\n';
+
+    appendFileSync(join(dir, `${today()}.jsonl`), line, { mode: 0o600 });
+    // Amortized retention sweep: 14-day window kept clean without a cron.
+    pruneOldShards(dir);
+  } catch { /* recorder must never throw */ }
+}
+
+/**
+ * Count hook errors with timestamp >= sinceMs. Used by `mem-cli stats` to
+ * surface a single self-observation line.
+ *
+ * Reads every shard (capped at 14 by retention). Each shard is JSONL — parse
+ * line-by-line and tolerate malformed lines (treat as zero contribution).
+ * Returns a plain count; per-scope breakdown is a follow-up if cardinality
+ * grows.
+ */
+export function countRecentHookErrors(runtimeDir, sinceMs) {
+  try {
+    if (!runtimeDir || typeof runtimeDir !== 'string') return 0;
+    const dir = hookErrorsDir(runtimeDir);
+    if (!existsSync(dir)) return 0;
+    let entries;
+    try { entries = readdirSync(dir); } catch { return 0; }
+    const cutoffIso = new Date(sinceMs).toISOString();
+    let count = 0;
+    for (const f of entries) {
+      if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)) continue;
+      let body;
+      try { body = readFileSync(join(dir, f), 'utf8'); } catch { continue; }
+      for (const line of body.split('\n')) {
+        if (!line) continue;
+        let parsed;
+        try { parsed = JSON.parse(line); } catch { continue; }
+        // ISO 8601 lexical ordering matches chronological ordering — no Date parse.
+        if (typeof parsed.ts === 'string' && parsed.ts >= cutoffIso) count++;
+      }
+    }
+    return count;
+  } catch { return 0; }
+}
+
+/** Test hook — current retention window (14 days, in ms). */
+export const HOOK_ERROR_RETENTION_MS = RETENTION_MS;

package/mem-cli.mjs

@@ -3,7 +3,7 @@
 // No MCP SDK or heavy deps — only imports schema.mjs and utils.mjs
 
 import { homedir } from 'os';
-import { ensureDb, DB_PATH, REGISTRY_DB_PATH } from './schema.mjs';
+import { ensureDb, DB_PATH, DB_DIR, REGISTRY_DB_PATH } from './schema.mjs';
 import { sanitizeFtsQuery, relaxFtsQueryToOr, truncate, typeIcon, inferProject, jaccardSimilarity, computeMinHash, estimateJaccardFromMinHash, scrubSecrets, cjkBigrams, isoWeekKey, COMPRESSED_PENDING_PURGE, SESS_BM25, DEFAULT_DECAY_HALF_LIFE_MS, notLowSignalTitleClause } from './utils.mjs';
 import { cjkPrecisionOk } from './nlp.mjs';
 import { extractCjkLikePatterns } from './nlp.mjs';
@@ -29,6 +29,7 @@ import { readFileSync, existsSync, readdirSync } from 'fs';
 // move each cmdXxx into its own cli/<cmd>.mjs; mem-cli.mjs becomes pure dispatch.
 import { parseArgs, out, fail, relativeTime, fmtDateShort, parseIdToken, formatProbeHints } from './cli/common.mjs';
 import { saveObservation } from './lib/save-observation.mjs';
+import { countRecentHookErrors } from './lib/hook-telemetry.mjs';
 import {
   insertDeferred, listOpenWithOrdinal, dropDeferred,
   resolveDeferredIds, closeDeferredItems,
@@ -1258,6 +1259,12 @@ async function cmdStats(db, args) {
     `SELECT COUNT(*) as c FROM observations WHERE superseded_at IS NOT NULL AND compressed_into IS NULL ${projectFilter}`
   ).get(...baseParams);
 
+  // Hook self-observation: count PreToolUse / Skill-bridge script failures
+  // recorded in the last 24h. Surfaces silent breakage (DB corruption,
+  // CC upstream field rename) that would otherwise stay invisible — the
+  // failure mode that left code-graph's matcher bug undetected for 10 sessions.
+  const hookErrors24h = countRecentHookErrors(join(DB_DIR, 'runtime'), now - 86400000);
+
   // Tier distribution (aligned with MCP mem_stats)
   const tierCtx = { now, currentProject: project || inferProject(), currentSessionId: '' };
   const tdParams = tierSqlParams(tierCtx);
@@ -1292,6 +1299,7 @@ async function cmdStats(db, args) {
         noise_ratio: Number(noiseRatio.toFixed(4)),
         compressed: compressedCount.c,
         superseded_only: supersededOnlyCount.c,
+        hook_errors_24h: hookErrors24h,
       },
       tier_distribution: {
         working: tierMap.working ?? 0,
@@ -1326,6 +1334,7 @@ async function cmdStats(db, args) {
   out(`  Avg importance: ${(avgImp.v ?? 1).toFixed(2)}`);
   out(`  Low-value (imp=1, never accessed, >30d): ${lowVal.c} (${(noiseRatio * 100).toFixed(1)}% noise)`);
   out(`  Compressed: ${compressedCount.c}`);
+  out(`  Hook errors (last 24h): ${hookErrors24h}${hookErrors24h > 0 ? `  ← tail ${join(DB_DIR, 'runtime/hook-errors')}` : ''}`);
   if (noiseRatio > 0.6) out('  ⚠️ High noise ratio — consider running mem compress');
   out('');
   // Tier counts only live (uncompressed, non-superseded) observations — surface the
@@ -2417,11 +2426,29 @@ function cmdCitationStats(db, args) {
      LIMIT 10
   `).all(cutoff);
 
+  // v34.x: surface pre-v34 data pollution. applyCitationDecay bumps cited_count
+  // and decay_seen_count atomically (same UPDATE statement), so the invariant
+  // cited_count <= decay_seen_count holds for every resolution this codepath
+  // performs. Yet a small set of obs violate it — these are pre-v34 rows
+  // where a backfill seeded cited_count without populating decay_seen_count.
+  // Without this note, those rows make per-project cite_pct >100% with no
+  // explanation. Cite rate stays unbiased for obs created after this commit.
+  const pollutedRows = db.prepare(`
+    SELECT COUNT(*) AS n FROM observations
+     WHERE cited_count > decay_seen_count
+       AND COALESCE(compressed_into, 0) = 0
+       AND superseded_at IS NULL
+  `).get();
+  const dataPollutionNote = pollutedRows.n > 0
+    ? `${pollutedRows.n} obs have cited_count > decay_seen_count (pre-v34 backfill — invariant holds for new data).`
+    : null;
+
   if (json) {
-    out(JSON.stringify({ window_days: days, per_project: perProject, decay_queue: decayQueue, promoted, demoted }, null, 2));
+    out(JSON.stringify({ window_days: days, per_project: perProject, decay_queue: decayQueue, promoted, demoted, data_pollution_note: dataPollutionNote }, null, 2));
     return;
   }
 
+  if (dataPollutionNote) out(`Note: ${dataPollutionNote}\n`);
   out(`Cite rate by project (last ${days}d, cited / decay-resolutions):`);
   for (const r of perProject) {
     const rate = r.resolved > 0 ? (r.cited * 100 / r.resolved).toFixed(1) + '%' : '—';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.75.0",
+  "version": "2.77.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "packageManager": "npm@10.9.2",
@@ -60,6 +60,7 @@
     "lib/summary-extractor.mjs",
     "lib/id-routing.mjs",
     "lib/err-sampler.mjs",
+    "lib/hook-telemetry.mjs",
     "lib/metrics.mjs",
     "lib/binding-probe.mjs",
     "lib/mem-override.mjs",

package/scripts/pre-skill-bridge.js

@@ -6,7 +6,11 @@
 import { existsSync, readFileSync } from 'fs';
 import { join, resolve, sep } from 'path';
 import { homedir } from 'os';
+import { recordHookError } from '../lib/hook-telemetry.mjs';
 
+// CLAUDE_MEM_DIR mirrors pre-tool-recall.js — one env var sandboxes everything.
+const DATA_DIR = process.env.CLAUDE_MEM_DIR || join(homedir(), '.claude-mem-lite');
+const RUNTIME_DIR = process.env.CLAUDE_MEM_RUNTIME_DIR || join(DATA_DIR, 'runtime');
 const REGISTRY_DB_PATH = join(homedir(), '.claude-mem-lite', 'resource-registry.db');
 const MANAGED_BASE = join(homedir(), '.claude-mem-lite');
 const MANAGED_MARKER = '/.claude-mem-lite/managed/';
@@ -24,7 +28,10 @@ try {
   try {
     const event = JSON.parse(input);
     skillName = event.tool_input?.skill;
-  } catch { process.exit(0); }
+  } catch (e) {
+    recordHookError('skill-bridge:json', e, RUNTIME_DIR, { inputLen: input.length });
+    process.exit(0);
+  }
 
   if (!skillName || typeof skillName !== 'string') process.exit(0);
 
@@ -37,7 +44,10 @@ try {
   try {
     db = new Database(REGISTRY_DB_PATH, { readonly: true });
     db.pragma('busy_timeout = 1000');
-  } catch { process.exit(0); }
+  } catch (e) {
+    recordHookError('skill-bridge:db-open', e, RUNTIME_DIR);
+    process.exit(0);
+  }
 
   try {
     // Query: find by name or invocation_name, ONLY if managed path
@@ -85,11 +95,13 @@ try {
         additionalContext,
       },
     }));
-  } catch {
-    // Silent failure — never block Skill tool
+  } catch (e) {
+    // Silent failure — never block Skill tool, but record for self-observation.
+    recordHookError('skill-bridge:query', e, RUNTIME_DIR, { skillName });
   } finally {
     try { db.close(); } catch {}
   }
-} catch {
-  // Top-level catch — exit 0 no matter what
+} catch (e) {
+  // Top-level catch — exit 0 no matter what, but record what slipped past.
+  try { recordHookError('skill-bridge:top', e, RUNTIME_DIR); } catch {}
 }

package/scripts/pre-tool-recall.js

@@ -8,6 +8,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
 import { basename, join } from 'path';
 import { homedir } from 'os';
 import { buildNotLowSignalSql } from '../lib/low-signal-patterns.mjs';
+import { recordHookError } from '../lib/hook-telemetry.mjs';
 
 // CLAUDE_MEM_DIR matches schema.mjs / main CLI — one env var sandboxes the
 // whole system. CLAUDE_MEM_DB_PATH / CLAUDE_MEM_RUNTIME_DIR remain as
@@ -87,9 +88,24 @@ try {
     filePath = event.tool_input?.file_path;
     sessionId = event.session_id || null;
     toolName = event.tool_name || null;
-  } catch { process.exit(0); }
+  } catch (e) {
+    recordHookError('pre-recall:json', e, RUNTIME_DIR, { inputLen: input.length });
+    process.exit(0);
+  }
 
-  if (!filePath) process.exit(0);
+  // Upstream-shape probe: hook ran but neither field nor input shape matches the
+  // contract we encode (event.tool_input.file_path, event.tool_name in
+  // Edit|Write|NotebookEdit|Read). Distinguishes "Claude Code renamed the field"
+  // from "event genuinely has no file_path" — without this trace, a CC upstream
+  // rename silently zeroes injection like code-graph's matcher bug.
+  if (!filePath) {
+    if (toolName && !['Edit', 'Write', 'NotebookEdit', 'Read'].includes(toolName)) {
+      recordHookError('pre-recall:unknown-tool', new Error(`tool_name=${toolName}`), RUNTIME_DIR, { toolName });
+    } else if (!toolName) {
+      recordHookError('pre-recall:no-toolname', new Error('event missing tool_name'), RUNTIME_DIR);
+    }
+    process.exit(0);
+  }
 
   // v2.34.6 Gap 3: Read-side recall with asymmetric quiet-mode. Reads have
   // lower per-event information value than Edits (passive observation, may
@@ -117,7 +133,10 @@ try {
   try {
     db = new Database(DB_PATH, { readonly: true });
     db.pragma('busy_timeout = 1000');
-  } catch { process.exit(0); }
+  } catch (e) {
+    recordHookError('pre-recall:db-open', e, RUNTIME_DIR);
+    process.exit(0);
+  }
 
   try {
     const project = inferProject();
@@ -258,11 +277,13 @@ try {
     // the per-filePath invariant that underpins Read→Edit dedup.
     cooldown[filePath] = now;
     writeCooldown(cooldownPath, cooldown, isSessionScoped);
-  } catch {
-    // Silent failure — never block editing
+  } catch (e) {
+    // Silent failure — never block editing, but record for self-observation.
+    recordHookError('pre-recall:query', e, RUNTIME_DIR, { filePath });
   } finally {
     try { db.close(); } catch {}
   }
-} catch {
-  // Top-level catch — exit 0 no matter what
+} catch (e) {
+  // Top-level catch — exit 0 no matter what, but record what slipped past.
+  try { recordHookError('pre-recall:top', e, RUNTIME_DIR); } catch {}
 }

package/source-files.mjs

@@ -43,6 +43,11 @@ export const SOURCE_FILES = [
   'lib/summary-extractor.mjs',
   'lib/id-routing.mjs',
   'lib/err-sampler.mjs',
+  // v2.76.x: unsampled hook-script failure log. Imported by
+  // scripts/pre-tool-recall.js + scripts/pre-skill-bridge.js (recorder)
+  // and mem-cli.mjs (countRecentHookErrors for `stats`). Missing from
+  // manifest → tarball ships hooks that ERR_MODULE_NOT_FOUND on every fire.
+  'lib/hook-telemetry.mjs',
   'lib/metrics.mjs',
   // v2.71.x: better-sqlite3 ABI probe + auto-rebuild. Shared by install.mjs
   // (post-`npm install` verify) and scripts/launch.mjs (pre-server-launch