lucid-mem 0.5.0 → 0.6.0

package/README.md

@@ -34,7 +34,7 @@ Memory lives in `~/agent-memory` — a real git repo you can read, diff, and poi
 
 ## Multi-repo memory
 
-One memory repo serves every project. `projects/<name>-<hash>.md` files key off the repo's origin remote, so clones share memory and same-named repos don't collide. Inside a known repo, `amr context` is project-scoped — the project's latest 25 plus linked siblings (up to 8 projects). Outside one, it shows global bullets plus the latest 25 pooled across all projects. `private` policy keeps a project out of transitive auto-attach and the pooled digest; `noglobal` keeps it out of the pooled digest only.
+One memory repo serves every project. `projects/<name>-<hash>.md` files key off the repo's origin remote, so clones share memory and same-named repos don't collide. Inside a known repo, `amr context` is project-scoped — the project's latest 25 plus linked siblings (up to 8 projects). Outside one, it shows global bullets plus the latest 25 pooled across all projects. `private` and `noglobal` policies keep a project out of the pooled digest (links still reach private projects — your link graph is your own curation).
 
 ## MCP
 

package/lib/cli.mjs

@@ -18,6 +18,7 @@ import {
 import { cmdDiscover } from './discover.mjs';
 import { cmdDream } from './dream.mjs';
 import { cmdHarvest } from './harvest.mjs';
+import { cmdCoverage } from './coverage.mjs';
 import { cmdDoctor, cmdInstall, cmdUninstall } from './install.mjs';
 import { cmdServe } from './serve.mjs';
 
@@ -41,6 +42,7 @@ usage: amr <command> [args] [flags]
   dream [--days N] [--fix] [--compact] [--keep N] [--json]   memory health; --fix cleans up, --compact archives overflow; --spin plays the animation
   harvest [--days N] [--json] [--extract] [--max N] [--min C]   list transcripts; --extract streams normalized user messages
   discover [--days N] [--min N] [--json]   rank repos by transcript mentions; shows memory coverage + suggested links
+  coverage [--days N] [--json] [--no-log]  save-rate metric: durable transcript signals vs memory repo
   install [--host a,b|all]      wire hooks + rules + MCP into detected agent hosts
   uninstall [--host a,b|all]    remove amr hooks + rules + MCP entries
   serve                         stdio MCP server (memory_context/add/search/update)
@@ -131,6 +133,8 @@ export function main(argv) {
         return cmdHarvest(flags);
       case 'discover':
         return cmdDiscover(flags);
+      case 'coverage':
+        return cmdCoverage(flags);
       case 'install':
         return cmdInstall(flags);
       case 'uninstall':

package/lib/commands.mjs

@@ -91,9 +91,9 @@ export function cmdContext(flags) {
 
   // Attach list: the current project, then its config links
   // (transitive, deduped, max 8 total) and any --also names.
-  // private blocks reachability from repos that never linked it:
-  // direct links (siblings) and explicit --also attach fine; it only
-  // refuses when reached transitively through another project's links.
+  // Links are the owner's curation — private projects attach fine
+  // through them. `private` only keeps a project out of the pooled
+  // outside digest, the one unlinked exposure path.
   const bySlug = new Map(listProjects(home).map((p) => [p.slug, p]));
   const attach = [];
   const seen = new Set();
@@ -103,17 +103,16 @@ export function cmdContext(flags) {
     attach.push({ slug, name: bySlug.get(slug)?.name || slug, linked });
   };
   if (proj) push(proj.slug, false);
-  const queue = (entry?.links || []).map((k) => ({ k, via: 'entry' }));
+  const queue = (entry?.links || []).map((k) => ({ k }));
   if (flags.also)
     queue.push(
       ...String(flags.also)
         .split(',')
-        .map((k) => ({ k, via: 'also' }))
+        .map((k) => ({ k }))
     );
-  const refused = [];
   const unknown = [];
   while (queue.length && attach.length < 8) {
-    const { k: raw, via } = queue.shift();
+    const { k: raw } = queue.shift();
     const key = String(raw).trim().toLowerCase();
     if (!key) continue;
     const slug = resolveProjectSlug(home, key);
@@ -123,13 +122,8 @@ export function cmdContext(flags) {
     }
     if (seen.has(slug)) continue;
     const e = projectEntry(cfg, { slug, name: key });
-    if (e?.private && via === 'transitive') {
-      refused.push(key);
-      continue;
-    }
     push(slug, true);
-    if (e?.links)
-      queue.push(...e.links.map((l) => ({ k: l, via: 'transitive' })));
+    if (e?.links) queue.push(...e.links.map((l) => ({ k: l })));
   }
 
   const lines = [`<agent-memory repo="${home}">`];
@@ -165,11 +159,6 @@ export function cmdContext(flags) {
       lines.push('', '## Recent project memory', recent.join('\n'));
   }
   if (indexLinks.length) lines.push('', '## Index', indexLinks.join('\n'));
-  if (refused.length)
-    lines.push(
-      '',
-      `<!-- ${refused.length} linked project(s) omitted by private policy: ${refused.join(', ')} -->`
-    );
   if (unknown.length)
     lines.push(
       '',

package/lib/coverage.mjs

@@ -0,0 +1,181 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { commit, ensureRepo, numFlag, withLock } from './core.mjs';
+import { collectSessions, extractMessages } from './harvest.mjs';
+
+const out = (s = '') => process.stdout.write(s + '\n');
+
+// What "should have been saved" looks like in a user message: durable
+// preferences, decisions, confirmations, and root-cause findings.
+// STRONG markers alone qualify; WEAK markers only in short messages
+// (long dumps happen to contain them by accident).
+const STRONG =
+  /\b(prefer|always|never|remember|note that|decided|instead of|rather than|default to|don['’]?t use|do not use|we use|sticking with|confirmed|turns out|workaround|the fix was|root cause|deprecated)\b/i;
+const WEAK = /\b(shipped|released|pinned|going with)\b/i;
+
+// Non-user prose that rides the user role in transcripts: tool errors,
+// context summaries, file/code dumps, listings, test output, diffs,
+// annotations, agent-monologue steps (antigravity mixes them into the
+// user slot).
+const DUMP =
+  /^(error:|context summary|#|```|\{|\[|\/|\d+\s|\d+[:|]|[-\w./]+ total \d+|total \d+ |diff |--- |@@ |index |[MDAR] \S+\.\w+|response annotations|base directory)/i;
+const heavyMarkup = (t) =>
+  (t.match(/[{}"|#*<>_=[\]\\`]/g) || []).length / t.length > 0.12;
+const FILE_REFS =
+  /\b[\w.-]+\.(?:go|tsx?|mjs|jsx?|cs|json|ya?ml|py|rs|cpp|c|h|png|jpe?g|dll|exe|log|toml|sql|ps1|sh)\b/g;
+// Source-code density: several of () ; {} // => == ++ --- is a paste
+// of code, not a user statement.
+const isCodey = (t) =>
+  (t.match(/[{}();]|\/\/|=>|={2,}|[+-]{3,}/g) || []).length >= 4;
+const isDump = (t) =>
+  DUMP.test(t) ||
+  heavyMarkup(t) ||
+  isCodey(t) ||
+  /[drwx-]{10}/.test(t) ||
+  /(\d+ (files?|tests?) (passed|failed))|passed \(\d+\)|RUN v\d|✓/.test(t) ||
+  /^(the user (is|asked|wants)|we (need|should|can|must|will)\b)/i.test(t) ||
+  (t.match(FILE_REFS) || []).length >= 3;
+const isDurable = (t) =>
+  !isDump(t) &&
+  (STRONG.test(t) || (t.length < 200 && WEAK.test(t)));
+
+const STOP = new Set(
+  'the a an and or but with this that from into your you for are was were have has been not all can will would could should about when what which there their they them then than also just like over under more some any each other such only very make made does done here need want keep lets'.split(
+    ' '
+  )
+);
+
+// Most distinctive terms first: code tokens and long words carry the
+// signal that a memory entry would share.
+const terms = (text) =>
+  [
+    ...new Set(
+      text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}_.\/-]+/gu, ' ')
+        .split(/\s+/)
+        .filter((w) => w.length >= 4 && !STOP.has(w))
+    ),
+  ]
+    .sort((a, b) => b.length - a.length)
+    .slice(0, 12);
+
+const normWords = (t) =>
+  t
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}_.\/-]+/gu, ' ')
+    .split(/\s+/)
+    .filter(Boolean);
+
+const corpusText = (home) => {
+  const parts = [];
+  (function walk(dir) {
+    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (e.name === '.git') continue;
+      const p = path.join(dir, e.name);
+      if (e.isDirectory()) walk(p);
+      else if (e.name.endsWith('.md'))
+        parts.push(normWords(fs.readFileSync(p, 'utf8')).join(' '));
+    }
+  })(home);
+  return ` ${parts.join(' ')} `;
+};
+
+// A durable signal counts as covered when a 4-word run of its content
+// appears verbatim in the corpus (near-verbatim save), or when ≥35% of
+// its distinctive terms appear (paraphrased save). Term hits are
+// prefix-aware at 6 chars so prefer matches prefers/preference.
+const covered = (text, corpus) => {
+  const content = normWords(text).filter(
+    (w) => w.length >= 4 && !STOP.has(w)
+  );
+  for (let i = 0; i + 4 <= content.length; i++)
+    if (corpus.includes(` ${content.slice(i, i + 4).join(' ')} `))
+      return true;
+  const t = terms(text);
+  if (!t.length) return false;
+  const hit = (w) =>
+    corpus.includes(` ${w} `) ||
+    corpus.includes(` ${w.slice(0, Math.min(6, w.length))}`);
+  const hits = t.filter(hit).length;
+  return hits / t.length >= 0.35;
+};
+
+export function cmdCoverage(flags) {
+  const home = ensureRepo();
+  const days = numFlag(flags.days ?? '7', 7);
+  const since = flags.since
+    ? new Date(flags.since).getTime()
+    : Date.now() - days * 864e5;
+  const max = numFlag(flags.max || '400', 400);
+
+  const found = collectSessions(since);
+  const msgs = extractMessages(found, { max, since });
+  const corpus = corpusText(home);
+
+  const seen = new Set();
+  const durable = [];
+  for (const m of msgs) {
+    const key = m.text.toLowerCase().replace(/\s+/g, ' ').trim();
+    if (seen.has(key) || !isDurable(m.text)) continue;
+    seen.add(key);
+    durable.push({ ...m, covered: covered(m.text, corpus) });
+  }
+
+  const hits = durable.filter((d) => d.covered).length;
+  const missed = durable.filter((d) => !d.covered);
+  const rate = durable.length ? Math.round((hits / durable.length) * 100) : null;
+
+  if (flags.json)
+    return out(
+      JSON.stringify(
+        {
+          days,
+          transcripts: found.length,
+          messages: msgs.length,
+          durable: durable.length,
+          covered: hits,
+          rate,
+          missed: missed.map((m) => ({
+            host: m.host,
+            date: m.mtime.slice(0, 10),
+            text: m.text.slice(0, 300),
+          })),
+        },
+        null,
+        2
+      )
+    );
+
+  if (!durable.length)
+    return out(`no durable signals found in ${found.length} transcript(s), ${days}d`);
+  out(
+    `coverage (${days}d): ${hits}/${durable.length} durable signals covered — save rate ${rate}%`
+  );
+  if (missed.length) {
+    out('missed:');
+    for (const m of missed.slice(0, 20))
+      out(`  [${m.host}] ${m.mtime.slice(0, 10)} ${m.text.slice(0, 140)}`);
+    if (missed.length > 20) out(`  … +${missed.length - 20} more`);
+  }
+  out(
+    'note: user-stated signals only; facts the agent learned silently are not observable from transcripts'
+  );
+
+  // Trend line: append this run so the metric has history.
+  if (!flags['no-log']) {
+    const log = path.join(home, 'dreams', 'coverage.jsonl');
+    const entry = {
+      ts: new Date().toISOString(),
+      days,
+      durable: durable.length,
+      covered: hits,
+      rate,
+    };
+    withLock(home, () => {
+      fs.mkdirSync(path.dirname(log), { recursive: true });
+      fs.appendFileSync(log, JSON.stringify(entry) + '\n');
+      commit(home, ['dreams/coverage.jsonl'], `Coverage ${rate}% (${hits}/${durable.length})`);
+    });
+  }
+}

package/lib/install.mjs

@@ -737,6 +737,19 @@ export function cmdDoctor() {
     }
     if (dead.length)
       out(`config      : ${dead.length} unresolvable project key(s): ${dead.join(', ')}`);
+    try {
+      const cov = fs
+        .readFileSync(path.join(homeDir, 'dreams', 'coverage.jsonl'), 'utf8')
+        .trim()
+        .split('\n')
+        .filter(Boolean)
+        .map((l) => JSON.parse(l));
+      const last = cov[cov.length - 1];
+      if (last?.rate != null)
+        out(
+          `save rate   : ${last.rate}% (${last.covered}/${last.durable} durable signals, ${last.days}d, ${last.ts.slice(0, 10)})`
+        );
+    } catch {}
     const missingPeers = (cfg.extraRepos || []).filter(
       (p) =>
         !fs.existsSync(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lucid-mem",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "lucid-mem (amr) - autonomous git-backed agent memory (Agent Memory Repo spec)",
   "type": "module",
   "bin": {

package/test/context.test.mjs

@@ -28,7 +28,7 @@ test('context in a repo shows its project memory', () => {
   assert.match(r.stdout, /app uses redis cache/);
 });
 
-test('link attaches a sibling project transitively; private blocks transitive only', () => {
+test('link attaches siblings and private transitively (links are owner curation)', () => {
   const home = tmpdir();
   const base = tmpdir();
   const repoA = initRepo(path.join(base, 'a'), 'https://github.com/acme/a.git');
@@ -45,13 +45,13 @@ test('link attaches a sibling project transitively; private blocks transitive on
   const inB = run(['context'], { amrHome: home, cwd: repoB });
   assert.match(inB.stdout, /a uses sqlite/);
 
-  // c links to b; a is private -> transitive refusal in c
+  // c links to b; a is private -> still attaches transitively, since
+  // the link chain is the owner's own curation
   run(['link', 'b'], { amrHome: home, cwd: repoC });
   run(['policy', 'a', 'private', 'true'], { amrHome: home, cwd: repoA });
   const inC = run(['context'], { amrHome: home, cwd: repoC });
   assert.match(inC.stdout, /b uses kafka/);
-  assert.match(inC.stdout, /omitted by private policy/);
-  assert.ok(!inC.stdout.includes('a uses sqlite'));
+  assert.match(inC.stdout, /a uses sqlite/);
 
   // explicit --also still attaches the private project
   const explicit = run(['context', '--also', 'a'], { amrHome: home, cwd: repoC });

package/test/coverage.test.mjs

@@ -0,0 +1,71 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { run, tmpdir } from './helpers.mjs';
+
+function setup() {
+  const amrHome = tmpdir();
+  const fakeHome = tmpdir();
+  assert.equal(run(['ensure'], { amrHome, fakeHome }).status, 0);
+  return { amrHome, fakeHome };
+}
+
+function claudeJsonl(fakeHome, lines) {
+  const dir = path.join(fakeHome, '.claude', 'projects', 'projx');
+  fs.mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, 'sess.jsonl');
+  fs.writeFileSync(
+    file,
+    lines
+      .map((text) =>
+        JSON.stringify({ type: 'user', message: { content: text } })
+      )
+      .join('\n')
+  );
+}
+
+test('coverage reports save rate and lists missed durable signals', () => {
+  const { amrHome, fakeHome } = setup();
+  run(['add', 'prefers tabs over spaces'], { amrHome, fakeHome });
+  claudeJsonl(fakeHome, [
+    // saved: near-verbatim in corpus
+    'I prefer tabs over spaces everywhere, remember that',
+    // durable but never saved
+    'we decided the cache TTL should always be 300 seconds',
+    // a code dump that merely contains a marker word — excluded
+    'const x = () => { if (a) { return b; } }; // never skip this check',
+    // ordinary chatter — not durable
+    'looks good, ship it',
+  ]);
+  const r = run(['coverage', '--days', '7', '--no-log'], {
+    amrHome,
+    fakeHome,
+  });
+  assert.equal(r.status, 0, r.stderr);
+  assert.match(r.stdout, /1\/2 durable signals covered — save rate 50%/);
+  assert.match(r.stdout, /cache TTL should always be 300/);
+  assert.ok(!r.stdout.includes('never skip this check'));
+});
+
+test('coverage logs a trend line by default; --no-log skips it', () => {
+  const { amrHome, fakeHome } = setup();
+  claudeJsonl(fakeHome, ['we decided the db migration runs weekly']);
+  const log = path.join(amrHome, 'dreams', 'coverage.jsonl');
+  run(['coverage', '--no-log'], { amrHome, fakeHome });
+  assert.ok(!fs.existsSync(log));
+  const r = run(['coverage'], { amrHome, fakeHome });
+  assert.equal(r.status, 0, r.stderr);
+  const entry = JSON.parse(readLines(log).at(-1));
+  assert.equal(entry.durable, 1);
+  assert.equal(entry.covered, 0);
+  assert.equal(entry.rate, 0);
+});
+
+function readLines(file) {
+  return fs
+    .readFileSync(file, 'utf8')
+    .trim()
+    .split('\n')
+    .filter(Boolean);
+}