claude-mem-lite 2.69.0 → 2.71.0

package/lib/scrub-record.mjs

@@ -0,0 +1,63 @@
+// claude-mem-lite: per-table scrub helper. Applies scrubSecrets to the known
+// text fields of a table row. Numeric / JSON-blob / id fields are passed
+// through untouched.
+//
+// Failsafe policy: when the table is unknown, scrub every string field by
+// default. Newly added tables stay safe even before TEXT_FIELDS_BY_TABLE is
+// updated — over-scrubbing is the safe direction; under-scrubbing leaks.
+//
+// JSON-stringified array fields (e.g. session_handoffs.key_files,
+// session_handoffs.match_keywords-when-array) are NOT listed here — running
+// scrubSecrets over the JSON string can rewrite quoted values and break
+// downstream JSON.parse. Pre-scrub each element upstream of the
+// JSON.stringify call instead.
+
+import { scrubSecrets } from '../secret-scrub.mjs';
+
+export const TEXT_FIELDS_BY_TABLE = {
+  observations: [
+    'title', 'subtitle', 'text', 'narrative',
+    'concepts', 'facts', 'lesson_learned', 'search_aliases',
+  ],
+  session_summaries: [
+    'request', 'investigated', 'learned',
+    'completed', 'next_steps', 'remaining_items', 'notes',
+    'lessons', 'key_decisions',
+  ],
+  session_handoffs: [
+    'working_on', 'completed', 'unfinished',
+    // Excluded:
+    //   key_files       — JSON.stringify(array); pre-scrub elements at call site
+    //   match_keywords  — currently a space-joined plain string; keeping it
+    //                     here would scrub safely, but the value is built from
+    //                     tokenizeHandoff() output (alphanumeric tokens only),
+    //                     so secrets cannot survive the upstream tokenizer.
+    //                     Excluded to avoid double-work + future-proof against
+    //                     a refactor that switches to JSON.stringify.
+    // key_decisions is kept: call site uses '\n'.join (plain string), and
+    // decision titles can carry secrets verbatim (LLM output).
+    'key_decisions',
+  ],
+};
+
+/**
+ * Scrub the text fields of a record before INSERT.
+ * Returns a shallow copy with string text-fields scrubbed; the input object
+ * is left untouched. Non-string values (numbers, null, JSON blobs the caller
+ * has already stringified) flow through unchanged.
+ */
+export function scrubRecord(table, row) {
+  if (!row || typeof row !== 'object') return row;
+  const fields = TEXT_FIELDS_BY_TABLE[table];
+  const out = { ...row };
+  if (fields) {
+    for (const f of fields) {
+      if (typeof out[f] === 'string') out[f] = scrubSecrets(out[f]);
+    }
+  } else {
+    for (const k of Object.keys(out)) {
+      if (typeof out[k] === 'string') out[k] = scrubSecrets(out[k]);
+    }
+  }
+  return out;
+}

package/lib/upgrade-banner.mjs

@@ -0,0 +1,31 @@
+// One-shot v2.70.0 upgrade banner.
+// Split out of hook.mjs because hook.mjs has module-level side effects
+// (notably `if (!event) process.exit(0)` at top level) that abort vitest
+// workers if imported directly from a test. See test
+// tests/hook-upgrade-banner.test.mjs.
+
+import { writeFileSync, existsSync } from 'fs';
+import { join } from 'path';
+
+/**
+ * One-shot stderr banner on first SessionStart after v2.70.0 upgrade.
+ * Notifies users that the `### Deferred Work` block now reads from the
+ * deferred_work table (not high-importance observations as in v2.69.x).
+ * Idempotent via a marker file in `runtimeDir`; subsequent calls in the
+ * same project are silent.
+ *
+ * @param {object} args
+ * @param {string} args.project Project name (used in banner + marker filename).
+ * @param {string} args.runtimeDir RUNTIME_DIR (test override; production passes hook-shared.RUNTIME_DIR).
+ */
+export function emitV270UpgradeBanner({ project, runtimeDir }) {
+  const marker = join(runtimeDir, `.deferred-block-migrated-${project}`);
+  if (existsSync(marker)) return;
+  process.stderr.write(
+    `[mem] v2.70.0 upgrade notice (project "${project}"): Deferred Work block now ` +
+    `backed by deferred_work table. To keep an obs visible there, run ` +
+    `\`claude-mem-lite defer add "<title>" --priority 3\`. ` +
+    `Pin to 2.69.x to revert.\n`
+  );
+  try { writeFileSync(marker, String(Date.now())); } catch { /* best-effort marker */ }
+}

package/mem-cli.mjs

@@ -14,6 +14,7 @@ import { autoBoostIfNeeded, reRankWithContext, markSuperseded } from './server-i
 import { searchObservationsHybrid, findFtsAnchor } from './search-engine.mjs';
 import { ensureRegistryDb, upsertResource } from './registry.mjs';
 import { searchResources } from './registry-retriever.mjs';
+import { scrubRecord } from './lib/scrub-record.mjs';
 import { optimizePreview, optimizeRun } from './hook-optimize.mjs';
 import { buildSessionContextLines } from './hook-context.mjs';
 import { cmdAdopt, cmdUnadopt } from './adopt-cli.mjs';
@@ -28,6 +29,10 @@ import { readFileSync, existsSync, readdirSync } from 'fs';
 // move each cmdXxx into its own cli/<cmd>.mjs; mem-cli.mjs becomes pure dispatch.
 import { parseArgs, out, fail, relativeTime, fmtDateShort, parseIdToken, formatProbeHints } from './cli/common.mjs';
 import { saveObservation } from './lib/save-observation.mjs';
+import {
+  insertDeferred, listOpenWithOrdinal, dropDeferred,
+  resolveDeferredIds, closeDeferredItems,
+} from './lib/deferred-work.mjs';
 
 // ─── Commands ────────────────────────────────────────────────────────────────
 
@@ -884,7 +889,7 @@ function cmdSave(db, args) {
   const { positional, flags } = parseArgs(args);
   const text = positional.join(' ');
   if (!text) {
-    fail('[mem] Usage: claude-mem-lite save "<text>" [--type T] [--title T] [--importance N] [--project P] [--files f1,f2] [--lesson T]');
+    fail('[mem] Usage: claude-mem-lite save "<text>" [--type T] [--title T] [--importance N] [--project P] [--files f1,f2] [--lesson T] [--closes-deferred 1,D#42]');
     return;
   }
 
@@ -914,15 +919,60 @@ function cmdSave(db, args) {
     return;
   }
 
-  const result = saveObservation(db, {
-    content: text,
-    title: flags.title,
-    type,
-    importance: rawImp,
-    project,
-    files: saveFiles,
-    lesson_learned: rawLesson,
-  });
+  // --closes-deferred parsing: accepts comma-separated mixed tokens
+  // ("1,D#42,3") with bare integers treated as ordinals and "D#N" as raw ids.
+  // We pre-parse tokens here (cheap, syntax-only) but defer resolveDeferredIds
+  // INTO the transaction, AFTER the dedup check. Resolving outside the
+  // transaction would throw on the duplicate-replay path: the previously-
+  // closed deferred row is no longer 'open', so ordinal/id resolution would
+  // crash even though the duplicate short-circuit makes closure a no-op.
+  // Resolving inside the dedup-gated branch keeps "save the same content
+  // twice" idempotent (mirrors server.mjs:934 dedup-skip-closure intent).
+  let closesTokens = null;
+  if (flags['closes-deferred'] !== undefined && flags['closes-deferred'] !== false) {
+    const raw = String(flags['closes-deferred']);
+    closesTokens = raw.split(',').map(t => t.trim()).filter(Boolean).map(t => {
+      return /^\d+$/.test(t) ? parseInt(t, 10) : t;
+    });
+    if (closesTokens.length === 0) {
+      fail('[mem] --closes-deferred requires at least one token (integer ordinal or D#N)');
+      return;
+    }
+  }
+
+  let result;
+  let closesIds = null;
+  try {
+    result = db.transaction(() => {
+      const r = saveObservation(db, {
+        content: text,
+        title: flags.title,
+        type,
+        importance: rawImp,
+        project,
+        files: saveFiles,
+        lesson_learned: rawLesson,
+      });
+      // Skip closure on dedup short-circuit — the obs row already exists, so
+      // the deferred item should NOT be re-closed by a duplicate save call.
+      // Resolving deferred ids only on the non-duplicate path keeps repeated
+      // save commands (with the same --closes-deferred) idempotent even after
+      // the deferred row has transitioned out of 'open'.
+      if (r.kind === 'duplicate') return r;
+      if (closesTokens) {
+        closesIds = resolveDeferredIds(db, project, closesTokens);
+        closeDeferredItems(db, closesIds, r.id);
+      }
+      return r;
+    })();
+  } catch (e) {
+    if (closesTokens) {
+      fail(`[mem] save with --closes-deferred failed: ${e.message}`);
+    } else {
+      fail(`[mem] save failed: ${e.message}`);
+    }
+    return;
+  }
 
   if (result.kind === 'duplicate') {
     out(`[mem] Skipped: similar to existing #${result.existingId}. Use "claude-mem-lite get ${result.existingId}" to review.`);
@@ -930,7 +980,108 @@ function cmdSave(db, args) {
   }
 
   const lessonNote = result.lessonCaptured ? ' 💡lesson captured' : '';
-  out(`[mem] Saved #${result.id} [${result.type}] "${truncate(result.title, 80)}" (project: ${result.project})${lessonNote}`);
+  const closedNote = closesIds && closesIds.length > 0
+    ? ` Closed: ${closesIds.map(i => `D#${i}`).join(', ')}.`
+    : '';
+  out(`[mem] Saved #${result.id} [${result.type}] "${truncate(result.title, 80)}" (project: ${result.project})${lessonNote}${closedNote}`);
+}
+
+// ─── cmdDefer (sub-dispatch: add | list | drop) ──────────────────────────────
+
+function cmdDefer(db, args) {
+  const sub = args[0];
+  const rest = args.slice(1);
+  switch (sub) {
+    case 'add':  cmdDeferAdd(db, rest); break;
+    case 'list': cmdDeferList(db, rest); break;
+    case 'drop': cmdDeferDrop(db, rest); break;
+    default:
+      fail('[mem] Usage: claude-mem-lite defer <add|list|drop> ...');
+      fail('[mem]   defer add "<title>" [--priority 1|2|3] [--detail T] [--files f1,f2] [--project P]');
+      fail('[mem]   defer list [--project P] [--limit N]');
+      fail('[mem]   defer drop <id-or-D#N> --reason "<reason>" [--project P]');
+  }
+}
+
+function cmdDeferAdd(db, args) {
+  const { positional, flags } = parseArgs(args);
+  const title = positional.join(' ').trim();
+  if (!title) {
+    fail('[mem] Usage: claude-mem-lite defer add "<title>" [--priority 1|2|3] [--detail T] [--files f1,f2] [--project P]');
+    return;
+  }
+  const priority = flags.priority !== undefined ? parseInt(flags.priority, 10) : 2;
+  if (![1, 2, 3].includes(priority)) {
+    fail(`[mem] Invalid --priority "${flags.priority}". Must be 1 (low), 2 (normal), or 3 (urgent).`);
+    return;
+  }
+  const project = flags.project ? resolveProject(db, flags.project) : inferProject();
+  const detail = typeof flags.detail === 'string' ? flags.detail : null;
+  const files = flags.files
+    ? flags.files.split(',').map(f => f.trim()).filter(Boolean)
+    : null;
+
+  let r;
+  try {
+    r = insertDeferred(db, { project, title, priority, detail, files });
+  } catch (e) {
+    fail(`[mem] defer add failed: ${e.message}`);
+    return;
+  }
+  // Compute the freshly-inserted row's ordinal for an immediately-actionable
+  // response ("ok, deferred this as item N"). Mirrors server.mjs:980.
+  const open = listOpenWithOrdinal(db, project, 50);
+  const ord = open.find(o => o.id === r.id)?.ordinal ?? '?';
+  out(`[mem] Deferred as D#${r.id} (item ${ord}) in project "${project}".`);
+}
+
+function cmdDeferList(db, args) {
+  const { flags } = parseArgs(args);
+  const project = flags.project ? resolveProject(db, flags.project) : inferProject();
+  const limit = parseIntFlag(flags.limit, { name: '--limit', defaultValue: 10, max: 100 });
+  const list = listOpenWithOrdinal(db, project, limit);
+  if (list.length === 0) {
+    out(`[mem] No open deferred items in project "${project}".`);
+    return;
+  }
+  out(`[mem] Open deferred items (project "${project}"):`);
+  for (const r of list) {
+    const pTag = r.priority === 3 ? '🔴' : r.priority === 1 ? '⚪' : '🟡';
+    out(`  ${r.ordinal}. ${pTag} [P${r.priority}] ${r.title} (D#${r.id})`);
+  }
+}
+
+function cmdDeferDrop(db, args) {
+  const { positional, flags } = parseArgs(args);
+  if (positional.length === 0) {
+    fail('[mem] Usage: claude-mem-lite defer drop <id-or-D#N> --reason "<reason>" [--project P]');
+    return;
+  }
+  const reason = flags.reason;
+  if (!reason || typeof reason !== 'string' || reason.trim().length === 0) {
+    fail('[mem] defer drop requires --reason "<non-empty string>"');
+    return;
+  }
+  const rawTok = positional[0];
+  // Accept both bare integer (ordinal) and "D#N" string. Mirrors the MCP
+  // mem_defer_drop input contract (server.mjs:1025) by using the same
+  // single-element resolveDeferredIds call.
+  const token = /^\d+$/.test(rawTok) ? parseInt(rawTok, 10) : rawTok;
+  const project = flags.project ? resolveProject(db, flags.project) : inferProject();
+
+  let realId;
+  try {
+    [realId] = resolveDeferredIds(db, project, [token]);
+  } catch (e) {
+    fail(`[mem] defer drop: ${e.message}`);
+    return;
+  }
+  const r = dropDeferred(db, realId, reason);
+  if (r.changed === 0) {
+    out(`[mem] D#${realId} was not in 'open' status — drop is a no-op.`);
+    return;
+  }
+  out(`[mem] Dropped D#${realId} in project "${project}". Reason: ${reason.trim()}`);
 }
 
 // N-1: Quality-focused stats for R-2 A/B baseline.
@@ -1615,8 +1766,11 @@ function cmdCompress(db, args) {
         VALUES (?, ?, ?, ?, ?, 'active')
       `).run(sessionId, sessionId, proj, now.toISOString(), now.getTime());
 
+      // Defense-in-depth: source rows already scrubbed at original ingest, but
+      // the new compressed narrative is constructed here and re-persisted.
+      const safe = scrubRecord('observations', { text: narrative, title, narrative });
       const summaryResult = insertSummary.run(
-        sessionId, proj, narrative, dominantType, title, narrative,
+        sessionId, proj, safe.text, dominantType, safe.title, safe.narrative,
         medianDate.toISOString(), medianEpoch
       );
       const summaryId = Number(summaryResult.lastInsertRowid);
@@ -2178,6 +2332,19 @@ Commands:
     --project P         Project name
     --files f1,f2       Comma-separated file paths
     --lesson T          Lesson learned (≤500 chars; alias: --lesson-learned)
+    --closes-deferred 1,D#42  Close deferred items in same transaction
+
+  defer <action>        First-class deferred work (v2.70+)
+    add "<title>"       Mark deferred work for next session
+      --priority N      1-3 (default 2)
+      --detail T        Constraint + why deferred
+      --files f1,f2     Comma-separated file paths
+      --project P       Project name
+    list                List open deferred items
+      --limit N         Max results (default 10)
+      --project P       Filter by project
+    drop <D#N|ordinal>  Drop a deferred item (no fix needed)
+      --reason "..."    Required audit trail
 
   delete <id1,id2,...>  Delete observations by ID
     --confirm           Execute deletion (preview by default)
@@ -2251,6 +2418,9 @@ Commands:
     remove              Remove resource --name N --resource-type T
     reindex             Rebuild FTS5 index
 
+  import-jsonl <file-or-dir>      Import Claude Code JSONL transcripts (cold-start backfill)
+    --project P         Project name (default: inferred from cwd)
+
   activity <action>     Non-memdir event log (v2.31) — bugfix/lesson/bug/discovery/etc.
     save --type T "<title>" [--body "<text>"] [--files f1,f2] [--file path] [--importance 1-3] [--project P]
     search "<query>"    Search events [--type T] [--limit N] [--project P]
@@ -2336,6 +2506,57 @@ async function cmdImport(argv) {
   }
 }
 
+// ─── Import (Claude Code JSONL transcript — cold-start backfill) ─────────────
+
+async function cmdImportJsonl(db, argv) {
+  const { positional, flags } = parseArgs(argv);
+  const target = positional[0];
+  if (!target) {
+    fail('[mem] Usage: claude-mem-lite import-jsonl <file-or-dir> [--project <name>]');
+    return;
+  }
+
+  const project = flags.project || inferProject();
+  const fs = await import('fs');
+  const { join: pjoin, resolve } = await import('path');
+  const abs = resolve(target);
+
+  let files = [];
+  let st;
+  try { st = fs.statSync(abs); }
+  catch (e) { fail(`[mem] Cannot stat ${abs}: ${e.message}`); return; }
+
+  if (st.isDirectory()) {
+    const walk = (dir) => {
+      for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+        const p = pjoin(dir, e.name);
+        if (e.isDirectory()) walk(p);
+        else if (e.isFile() && p.endsWith('.jsonl')) files.push(p);
+      }
+    };
+    walk(abs);
+  } else {
+    files = [abs];
+  }
+
+  if (files.length === 0) { out('[mem] No .jsonl files found.'); return; }
+
+  const { importJsonl } = await import('./lib/import-jsonl.mjs');
+  let totalPrompts = 0, totalObs = 0, totalSkip = 0, totalOrphans = 0;
+  for (const f of files) {
+    const r = await importJsonl(db, f, { project });
+    totalPrompts += r.prompts;
+    totalObs += r.observations;
+    totalSkip += r.skipped;
+    totalOrphans += r.orphans || 0;
+    out(`[mem] ${f}: +${r.prompts} prompts, +${r.observations} observations, ${r.orphans || 0} orphan tool_use, ${r.skipped} skipped`);
+  }
+  out(`[mem] Total: ${totalPrompts} prompts, ${totalObs} observations, ${totalOrphans} orphan tool_use, ${totalSkip} skipped from ${files.length} file(s).`);
+  if (totalPrompts > 0 || totalObs > 0) {
+    out(`[mem] Try: claude-mem-lite recent 5 --project ${project}`);
+  }
+}
+
 // ─── Enrich ─────────────────────────────────────────────────────────────────
 
 async function cmdEnrich(argv) {
@@ -2506,6 +2727,7 @@ export async function run(argv) {
       case 'get':       cmdGet(db, cmdArgs); break;
       case 'timeline':  cmdTimeline(db, cmdArgs); break;
       case 'save':      cmdSave(db, cmdArgs); break;
+      case 'defer':     cmdDefer(db, cmdArgs); break;
       case 'delete':    cmdDelete(db, cmdArgs); break;
       case 'update':    cmdUpdate(db, cmdArgs); break;
       case 'export':    cmdExport(db, cmdArgs); break;
@@ -2518,6 +2740,7 @@ export async function run(argv) {
       case 'browse':    cmdBrowse(db, cmdArgs); break;
       case 'registry':  cmdRegistry(db, cmdArgs); break;
       case 'import':    await cmdImport(cmdArgs); break;
+      case 'import-jsonl': await cmdImportJsonl(db, cmdArgs); break;
       case 'enrich':    await cmdEnrich(cmdArgs); break;
       case 'doctor':    await cmdDoctor(db, cmdArgs); break;
       case 'activity':  await cmdActivity(db, cmdArgs); break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.69.0",
+  "version": "2.71.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "packageManager": "npm@10.9.2",
@@ -39,6 +39,7 @@
     "hook-handoff.mjs",
     "hook-update.mjs",
     "hook-optimize.mjs",
+    "hook-precompact.mjs",
     "plugin-cache-guard.mjs",
     "memdir.mjs",
     "adopt-content.mjs",
@@ -62,6 +63,10 @@
     "lib/metrics.mjs",
     "lib/mem-override.mjs",
     "lib/save-observation.mjs",
+    "lib/deferred-work.mjs",
+    "lib/upgrade-banner.mjs",
+    "lib/scrub-record.mjs",
+    "lib/import-jsonl.mjs",
     "cli/common.mjs",
     "cli/fts-check.mjs",
     "cli/doctor.mjs",

package/schema.mjs

@@ -40,7 +40,12 @@ export const REGISTRY_DB_PATH = join(DB_DIR, 'resource-registry.db');
 // DROP+CREATE so DBs that picked up the strict v29 trigger get the UUID-
 // gated body. Required because `CREATE TRIGGER IF NOT EXISTS` is a no-op
 // when the trigger already exists, even with a different body.
-export const CURRENT_SCHEMA_VERSION = 30;
+// v31 (v2.70.0): deferred_work table — first-class carry-forward surface.
+// Decoupled from observations: different decay semantics (no time decay; older
+// items rank HIGHER as tech debt accumulates), different lifecycle (mutable
+// status open→done|dropped vs immutable obs). Closure tied to obs via
+// closed_by_obs_id FK with ON DELETE SET NULL (audit trail preserved).
+export const CURRENT_SCHEMA_VERSION = 31;
 
 const CORE_SCHEMA = `
   CREATE TABLE IF NOT EXISTS sdk_sessions (
@@ -541,6 +546,33 @@ export function initSchema(db) {
     )
   `);
 
+  // ─── v31 (v2.70.0): deferred_work — carry-forward TODOs ─────────────────────
+  // Independent table because decay semantics are inverted (older = higher
+  // priority signal) and lifecycle is mutable (status flips). Project-scoped
+  // queries; no FTS5 (per-project N expected ≪ 100). Idempotent migration.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS deferred_work (
+      id                INTEGER PRIMARY KEY AUTOINCREMENT,
+      project           TEXT    NOT NULL,
+      title             TEXT    NOT NULL,
+      detail            TEXT,
+      priority          INTEGER NOT NULL DEFAULT 2,
+      status            TEXT    NOT NULL DEFAULT 'open' CHECK(status IN ('open','done','dropped')),
+      created_at_epoch  INTEGER NOT NULL,
+      closed_at_epoch   INTEGER,
+      closed_by_obs_id  INTEGER REFERENCES observations(id) ON DELETE SET NULL,
+      drop_reason       TEXT,
+      source_session_id TEXT,
+      source_prompt_id  INTEGER REFERENCES user_prompts(id) ON DELETE SET NULL,
+      files             TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_deferred_open
+      ON deferred_work(project, priority DESC, created_at_epoch ASC)
+      WHERE status = 'open';
+    CREATE INDEX IF NOT EXISTS idx_deferred_closed_by
+      ON deferred_work(closed_by_obs_id) WHERE closed_by_obs_id IS NOT NULL;
+  `);
+
   // Record schema version for fast-path on subsequent calls
   db.exec('CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)');
   db.transaction(() => {