@lmaksym/agent-mem 0.1.0

package/src/core/reflect-defrag.js

@@ -0,0 +1,287 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { readContextFile, writeContextFile, parseFrontmatter } from './fs.js';
+import { readConfig } from './config.js';
+import { commitContext } from './git.js';
+import { getReflectionFiles } from './reflect-gather.js';
+
+/**
+ * Tokenize a string into a set of lowercase words.
+ */
+function wordSet(text) {
+  return new Set(
+    text
+      .toLowerCase()
+      .replace(/[^a-z0-9\s]/g, ' ')
+      .split(/\s+/)
+      .filter((w) => w.length > 2),
+  );
+}
+
+/**
+ * Jaccard similarity between two word sets.
+ */
+function jaccard(setA, setB) {
+  if (setA.size === 0 && setB.size === 0) return 0;
+  let intersection = 0;
+  for (const w of setA) {
+    if (setB.has(w)) intersection++;
+  }
+  const union = setA.size + setB.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/**
+ * Parse entry lines from a memory file. Returns array of { line, date, text }.
+ */
+function parseEntries(content) {
+  if (!content) return [];
+  const entries = [];
+  const lines = content.split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const match = lines[i].match(/^- \[(\d{4}-\d{2}-\d{2})[\s\d:]*\]\s*(.+)/);
+    if (match) {
+      entries.push({ line: i + 1, date: match[1], text: match[2] });
+    }
+  }
+  return entries;
+}
+
+/**
+ * Get all text from reflections that "reaffirm" entries (Decisions Validated, Patterns Identified).
+ */
+function getReaffirmedText(ctxDir) {
+  const files = getReflectionFiles(ctxDir);
+  const texts = [];
+  for (const f of files) {
+    const raw = readContextFile(ctxDir, f);
+    if (!raw) continue;
+    // Extract relevant sections
+    const content = parseFrontmatter(raw).content;
+    const sectionMatch = content.match(
+      /## (?:Patterns Identified|Decisions Validated)\n([\s\S]*?)(?=\n## |\n$|$)/g,
+    );
+    if (sectionMatch) {
+      texts.push(...sectionMatch);
+    }
+  }
+  return texts.join('\n').toLowerCase();
+}
+
+/**
+ * Run defrag analysis on memory files.
+ * Returns { oversized, duplicates, stale, structural, health, issueCount }.
+ */
+export function analyzeDefrag(ctxDir) {
+  const config = readConfig(ctxDir);
+  const threshold = config.reflection?.defrag_threshold || 50;
+  const sizeKb = config.reflection?.defrag_size_kb || 10;
+  const staleDays = config.reflection?.stale_days || 30;
+
+  const memDir = join(ctxDir, 'memory');
+  if (!existsSync(memDir)) {
+    return {
+      oversized: [],
+      duplicates: [],
+      stale: [],
+      structural: [],
+      health: 'GOOD',
+      issueCount: 0,
+    };
+  }
+
+  const files = readdirSync(memDir)
+    .filter((n) => n.endsWith('.md') && !n.startsWith('.'))
+    .sort();
+
+  const oversized = [];
+  const duplicates = [];
+  const stale = [];
+  const structural = [];
+
+  const reaffirmed = getReaffirmedText(ctxDir);
+  const now = new Date();
+  const staleThreshold = new Date(now.getTime() - staleDays * 24 * 60 * 60 * 1000);
+
+  for (const name of files) {
+    const relPath = `memory/${name}`;
+    const content = readContextFile(ctxDir, relPath);
+    if (!content) continue;
+
+    const stat = statSync(join(memDir, name));
+    const entries = parseEntries(content);
+    const { description } = parseFrontmatter(content);
+
+    // Check oversized
+    if (entries.length > threshold || stat.size > sizeKb * 1024) {
+      oversized.push({
+        file: relPath,
+        entries: entries.length,
+        size: stat.size,
+        sizeFormatted: `${(stat.size / 1024).toFixed(1)}KB`,
+      });
+    }
+
+    // Check structural issues
+    if (!description && entries.length > 0) {
+      structural.push({ file: relPath, issue: 'missing frontmatter (description)' });
+    }
+    if (entries.length === 0 && content.trim().split('\n').length <= 6) {
+      structural.push({ file: relPath, issue: 'empty (no entries)' });
+    }
+
+    // Check duplicates within the same file
+    for (let i = 0; i < entries.length; i++) {
+      const wordsI = wordSet(entries[i].text);
+      for (let j = i + 1; j < entries.length; j++) {
+        const wordsJ = wordSet(entries[j].text);
+        const sim = jaccard(wordsI, wordsJ);
+        if (sim >= 0.6) {
+          duplicates.push({
+            file: relPath,
+            lineA: entries[i].line,
+            lineB: entries[j].line,
+            textA: entries[i].text.slice(0, 80),
+            textB: entries[j].text.slice(0, 80),
+            similarity: sim,
+          });
+        }
+      }
+    }
+
+    // Check stale entries
+    for (const entry of entries) {
+      const entryDate = new Date(entry.date);
+      if (entryDate < staleThreshold) {
+        // Check if this entry was reaffirmed in any reflection
+        const entryLower = entry.text.toLowerCase();
+        const isReaffirmed = reaffirmed.includes(entryLower.slice(0, 30));
+        if (!isReaffirmed) {
+          stale.push({
+            file: relPath,
+            line: entry.line,
+            date: entry.date,
+            text: entry.text.slice(0, 80),
+          });
+        }
+      }
+    }
+  }
+
+  const issueCount = oversized.length + duplicates.length + stale.length + structural.length;
+  const health = issueCount === 0 ? 'GOOD' : issueCount <= 3 ? 'FAIR' : 'NEEDS_ATTENTION';
+
+  return { oversized, duplicates, stale, structural, health, issueCount };
+}
+
+/**
+ * Format defrag analysis as output text.
+ */
+export function formatDefragOutput(analysis, isDryRun) {
+  const lines = [];
+  const marker = isDryRun ? ' (dry run — no changes)' : '';
+  lines.push(`🔧 DEFRAG ANALYSIS${marker}`);
+  lines.push('');
+
+  // Oversized
+  if (analysis.oversized.length > 0) {
+    lines.push('OVERSIZED FILES:');
+    for (const o of analysis.oversized) {
+      lines.push(`  ${o.file} — ${o.entries} entries, ${o.sizeFormatted}`);
+      lines.push(`  ⤷ Suggest: split by domain or time period`);
+    }
+    lines.push('');
+  }
+
+  // Duplicates
+  if (analysis.duplicates.length > 0) {
+    lines.push('POTENTIAL DUPLICATES:');
+    for (const d of analysis.duplicates) {
+      lines.push(
+        `  ${d.file} line ${d.lineA} ↔ line ${d.lineB} (${(d.similarity * 100).toFixed(0)}% similar)`,
+      );
+      lines.push(`    "${d.textA}"`);
+      lines.push(`    "${d.textB}"`);
+      lines.push(`  ⤷ Suggest: merge into single, more specific entry`);
+    }
+    lines.push('');
+  }
+
+  // Stale
+  if (analysis.stale.length > 0) {
+    lines.push(`STALE ENTRIES (>${analysis.stale[0] ? '' : ''}30 days, unreferenced):`);
+    for (const s of analysis.stale) {
+      lines.push(`  ${s.file} line ${s.line} — [${s.date}] "${s.text}"`);
+    }
+    lines.push(`  ⤷ Suggest: archive or remove if no longer relevant`);
+    lines.push('');
+  }
+
+  // Structural
+  if (analysis.structural.length > 0) {
+    lines.push('STRUCTURAL ISSUES:');
+    for (const s of analysis.structural) {
+      lines.push(`  ${s.file} — ${s.issue}`);
+    }
+    lines.push('');
+  }
+
+  if (analysis.issueCount === 0) {
+    lines.push('No issues found.');
+    lines.push('');
+  }
+
+  lines.push(`Memory health: ${analysis.health} (${analysis.issueCount} issues)`);
+  lines.push('');
+
+  // Instructions
+  lines.push('═══ INSTRUCTIONS ═══');
+  lines.push('To act on suggestions:');
+  lines.push('  amem write memory/<file>.md "<cleaned content>"');
+  lines.push('  amem commit "defrag: reorganized memory files"');
+  lines.push('');
+  lines.push('Or run a full reflect cycle to address holistically:');
+  lines.push('  amem reflect gather --deep');
+
+  return lines.join('\n');
+}
+
+/**
+ * Apply non-destructive stale markers to flagged entries.
+ * Returns count of entries marked.
+ */
+export function applyStaleMarkers(ctxDir, staleEntries) {
+  if (!staleEntries.length) return 0;
+
+  const today = new Date().toISOString().slice(0, 10);
+  let marked = 0;
+
+  // Group by file
+  const byFile = {};
+  for (const entry of staleEntries) {
+    if (!byFile[entry.file]) byFile[entry.file] = [];
+    byFile[entry.file].push(entry);
+  }
+
+  for (const [file, entries] of Object.entries(byFile)) {
+    let content = readContextFile(ctxDir, file);
+    if (!content) continue;
+
+    const lines = content.split('\n');
+    // Process in reverse order so line numbers stay valid
+    const lineNums = entries.map((e) => e.line).sort((a, b) => b - a);
+
+    for (const lineNum of lineNums) {
+      const idx = lineNum - 1;
+      if (idx >= 0 && idx < lines.length && !lines[idx].includes('[STALE]')) {
+        // Insert stale marker after the entry line
+        lines.splice(idx + 1, 0, `- [${today} STALE] ^^^ flagged by reflection — may be outdated`);
+        marked++;
+      }
+    }
+
+    writeContextFile(ctxDir, file, lines.join('\n'));
+  }
+
+  return marked;
+}

package/src/core/reflect-gather.js

@@ -0,0 +1,360 @@
+import { existsSync, readdirSync, statSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { readContextFile, writeContextFile, parseFrontmatter, listFiles } from './fs.js';
+import { readConfig } from './config.js';
+import {
+  gitLog,
+  gitDiffStat,
+  gitDiffFiles,
+  commitCountSince,
+  firstCommit,
+  lastCommit,
+} from './git.js';
+
+/**
+ * Determine the "since" reference for the reflection window.
+ * Priority: --since flag > last reflection's last_commit_hash > first commit
+ */
+export function resolveSinceRef(ctxDir, flags) {
+  // Explicit --since flag
+  if (flags.since) return flags.since;
+
+  // Check .reflect-state.json for last saved state
+  const stateRaw = readContextFile(ctxDir, '.reflect-state.json');
+  if (stateRaw) {
+    try {
+      const state = JSON.parse(stateRaw);
+      if (state.last_commit_hash) return state.last_commit_hash;
+    } catch {}
+  }
+
+  // Check most recent reflection file's frontmatter
+  const reflections = getReflectionFiles(ctxDir);
+  if (reflections.length > 0) {
+    const latest = readContextFile(ctxDir, reflections[reflections.length - 1]);
+    if (latest) {
+      const { description } = parseFrontmatter(latest);
+      // parseFrontmatter gives us description but we need the raw frontmatter
+      const match = latest.match(/^---\n([\s\S]*?)\n---/);
+      if (match) {
+        const hashMatch = match[1].match(/last_commit_hash:\s*['"]?(\w+)['"]?/);
+        if (hashMatch) return hashMatch[1];
+      }
+    }
+  }
+
+  // Fall back to first commit
+  return firstCommit(ctxDir);
+}
+
+/**
+ * Get sorted list of reflection file paths (relative to ctxDir).
+ */
+export function getReflectionFiles(ctxDir) {
+  const dir = join(ctxDir, 'reflections');
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((n) => n.endsWith('.md') && !n.startsWith('.'))
+    .sort()
+    .map((n) => `reflections/${n}`);
+}
+
+/**
+ * Count entries in a memory file (lines matching `^- [` or `^### [`).
+ */
+function countEntries(content) {
+  if (!content) return 0;
+  return content.split('\n').filter((l) => /^- \[/.test(l) || /^### \[/.test(l)).length;
+}
+
+/**
+ * Get last entry date from a memory file.
+ */
+function lastEntryDate(content) {
+  if (!content) return null;
+  const lines = content.split('\n').filter((l) => /^- \[/.test(l) || /^### \[/.test(l));
+  if (!lines.length) return null;
+  const match = lines[lines.length - 1].match(/^(?:- |### )\[(\d{4}-\d{2}-\d{2})/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Analyze all memory files. Returns array of { file, entries, size, lastDate, content }.
+ */
+function analyzeMemoryFiles(ctxDir) {
+  const memDir = join(ctxDir, 'memory');
+  if (!existsSync(memDir)) return [];
+
+  return readdirSync(memDir)
+    .filter((n) => n.endsWith('.md') && !n.startsWith('.'))
+    .sort()
+    .map((name) => {
+      const content = readContextFile(ctxDir, `memory/${name}`);
+      const stat = statSync(join(memDir, name));
+      return {
+        file: `memory/${name}`,
+        entries: countEntries(content),
+        size: stat.size,
+        lastDate: lastEntryDate(content),
+        content,
+      };
+    });
+}
+
+/**
+ * Analyze branch lifecycle. Returns { active, merged }.
+ */
+function analyzeBranches(ctxDir) {
+  const branchDir = join(ctxDir, 'branches');
+  if (!existsSync(branchDir)) return { active: [], merged: [] };
+
+  const config = readConfig(ctxDir);
+  const active = [];
+  const merged = [];
+
+  for (const name of readdirSync(branchDir).filter((n) => !n.startsWith('.'))) {
+    const purposeRaw = readContextFile(ctxDir, `branches/${name}/purpose.md`);
+    const purpose = purposeRaw
+      ? purposeRaw
+          .split('\n')
+          .filter((l) => l.trim() && !l.startsWith('#') && !l.startsWith('---'))[0]
+          ?.trim() || name
+      : name;
+
+    // Check if branch was merged (look for merge entry in decisions.md)
+    const decisions = readContextFile(ctxDir, 'memory/decisions.md') || '';
+    const mergePattern = new RegExp(`Merged branch: ${name}`, 'i');
+    if (mergePattern.test(decisions)) {
+      merged.push({ name, purpose });
+    } else {
+      active.push({ name, purpose, isCurrent: config.branch === name });
+    }
+  }
+
+  return { active, merged };
+}
+
+/**
+ * Get the last reflection summary text.
+ */
+function getLastReflectionSummary(ctxDir) {
+  const files = getReflectionFiles(ctxDir);
+  if (!files.length) return null;
+
+  const content = readContextFile(ctxDir, files[files.length - 1]);
+  if (!content) return null;
+
+  const { content: body } = parseFrontmatter(content);
+  // Extract Summary section
+  const match = body.match(/## Summary\n([\s\S]*?)(?=\n## |\n$|$)/);
+  const summary = match ? match[1].trim() : null;
+
+  // Extract frontmatter date
+  const dateMatch = content.match(/date:\s*['"]?(\d{4}-\d{2}-\d{2})['"]?/);
+  const date = dateMatch
+    ? dateMatch[1]
+    : files[files.length - 1].replace('reflections/', '').replace('.md', '');
+
+  return { date, file: files[files.length - 1], summary };
+}
+
+/**
+ * Format file size for display.
+ */
+function formatSize(bytes) {
+  if (bytes < 1024) return `${bytes}B`;
+  return `${(bytes / 1024).toFixed(1)}KB`;
+}
+
+/**
+ * Gather all reflection data and format the prompt.
+ */
+export function gatherReflectionPrompt(ctxDir, flags) {
+  const sinceRef = resolveSinceRef(ctxDir, flags);
+  const totalCommits = commitCountSince(ctxDir, sinceRef);
+
+  // Early exit if no activity
+  if (totalCommits === 0) {
+    return { empty: true, sinceRef };
+  }
+
+  const commits = gitLog(ctxDir, sinceRef, 50);
+  const diffStats = gitDiffStat(ctxDir, sinceRef);
+  const memoryFiles = analyzeMemoryFiles(ctxDir);
+  const branches = analyzeBranches(ctxDir);
+  const lastReflection = getLastReflectionSummary(ctxDir);
+  const last = lastCommit(ctxDir);
+
+  // Determine window dates
+  const windowEnd = new Date().toISOString().slice(0, 10);
+  const windowStart =
+    commits.length > 0 ? commits[commits.length - 1].date?.slice(0, 10) || 'unknown' : 'unknown';
+
+  // Write breadcrumb state for save phase
+  const state = {
+    window_start: windowStart,
+    window_end: windowEnd,
+    commits_reviewed: totalCommits,
+    last_commit_hash: last?.hash || null,
+    since_ref: sinceRef,
+    gathered_at: new Date().toISOString(),
+  };
+  writeContextFile(ctxDir, '.reflect-state.json', JSON.stringify(state, null, 2));
+
+  // Build prompt
+  const lines = [];
+
+  lines.push('🔍 REFLECTION INPUT');
+  lines.push(`Window: ${windowStart} → ${windowEnd} (${totalCommits} commits)`);
+  lines.push(
+    `Last reflection: ${lastReflection ? `${lastReflection.date} (${lastReflection.file})` : 'none (first reflection)'}`,
+  );
+  lines.push('');
+
+  // === RECENT ACTIVITY ===
+  lines.push('═══ RECENT ACTIVITY ═══');
+  lines.push('');
+
+  lines.push(`COMMITS (${totalCommits}):`);
+  for (let i = 0; i < commits.length; i++) {
+    const c = commits[i];
+    lines.push(`  ${i + 1}. ${c.hash} | ${c.message} | ${c.date?.slice(0, 16) || ''}`);
+  }
+  if (totalCommits > 50) {
+    lines.push(`  ... and ${totalCommits - 50} earlier commits`);
+  }
+  lines.push('');
+
+  if (diffStats.length > 0) {
+    lines.push('FILES CHANGED:');
+    for (const d of diffStats) {
+      lines.push(`  ${d.file}  +${d.added} -${d.removed} lines`);
+    }
+    lines.push('');
+  }
+
+  // Deep mode: include full diffs for memory files
+  if (flags.deep && sinceRef) {
+    const memoryDiff = gitDiffFiles(ctxDir, sinceRef, 'memory/');
+    if (memoryDiff) {
+      lines.push('MEMORY DIFFS (--deep):');
+      lines.push(memoryDiff);
+      lines.push('');
+    }
+  }
+
+  // Branches
+  if (branches.merged.length > 0 || branches.active.length > 0) {
+    lines.push('BRANCHES:');
+    for (const b of branches.merged) {
+      lines.push(`  MERGED: ${b.name} → "${b.purpose}"`);
+    }
+    for (const b of branches.active) {
+      lines.push(`  ACTIVE: ${b.name}${b.isCurrent ? ' *' : ''} → "${b.purpose}"`);
+    }
+    lines.push('');
+  }
+
+  // === CURRENT MEMORY STATE ===
+  lines.push('═══ CURRENT MEMORY STATE ═══');
+  lines.push('');
+
+  if (memoryFiles.length === 0) {
+    lines.push('(no memory files yet)');
+  } else {
+    for (const mf of memoryFiles) {
+      lines.push(`${mf.file} (${mf.entries} entries, ${formatSize(mf.size)}):`);
+      if (mf.entries === 0) {
+        lines.push('  (empty)');
+      } else {
+        // Show all entries for cross-reference (bullet entries and lesson headings)
+        const entryLines = mf.content
+          .split('\n')
+          .filter((l) => /^- \[/.test(l) || /^### \[/.test(l));
+        for (const entry of entryLines) {
+          lines.push(`  ${entry}`);
+        }
+      }
+      lines.push('');
+    }
+  }
+
+  // === LAST REFLECTION ===
+  lines.push('═══ LAST REFLECTION ═══');
+  lines.push('');
+  if (lastReflection) {
+    lines.push(`(from ${lastReflection.file})`);
+    lines.push(lastReflection.summary || '(no summary)');
+  } else {
+    lines.push('First reflection — no prior context.');
+    lines.push('');
+    lines.push('This is your first reflection for this project. Focus on:');
+    lines.push('1. Are the memory entries so far capturing the RIGHT things?');
+    lines.push('2. Are any decisions already outdated?');
+    lines.push('3. What implicit knowledge about this project have you NOT written down?');
+    lines.push('4. Are the system/ files (conventions, project overview) still accurate?');
+  }
+  lines.push('');
+
+  // === COMPACTION MODE ===
+  if (flags.compaction) {
+    lines.push('═══ COMPACTION MODE ═══');
+    lines.push('Your context window is filling up. Focus on:');
+    lines.push('1. Summarize work into concise status (not deep analysis)');
+    lines.push('2. Identify memory entries to archive');
+    lines.push('3. Identify system/ files to shorten or unpin');
+    lines.push('4. After saving, consider unpinning less critical system files');
+    lines.push('');
+  }
+
+  // === REFLECTION QUESTIONS ===
+  lines.push('═══ REFLECTION QUESTIONS ═══');
+  lines.push('');
+  lines.push('1. PATTERNS: What recurring approaches or successful strategies emerged?');
+  lines.push(
+    '2. LESSONS: Were any problems debugged/fixed? Record each as a problem→resolution pair.',
+  );
+  lines.push('3. CONTRADICTIONS: Do any new entries conflict with existing ones?');
+  lines.push('4. GAPS: What important context is NOT yet captured?');
+  lines.push('5. STALE: Are any existing entries outdated?');
+  lines.push('6. THEMES: What overarching directions are emerging?');
+  lines.push('');
+
+  // === INSTRUCTIONS ===
+  lines.push('═══ INSTRUCTIONS ═══');
+  lines.push('');
+  lines.push('After reasoning, call:');
+  lines.push('  amem reflect save --content "YOUR_REFLECTION"');
+  lines.push('');
+  lines.push('Use this format:');
+  lines.push('');
+  lines.push('## Patterns Identified');
+  lines.push('- <pattern>');
+  lines.push('');
+  lines.push('## Decisions Validated');
+  lines.push('- <decision confirmed by recent work>');
+  lines.push('');
+  lines.push('## Contradictions Found');
+  lines.push('- <what conflicts, and resolution>');
+  lines.push('');
+  lines.push('## Stale Entries');
+  lines.push('- <file>: <entry to flag>');
+  lines.push('');
+  lines.push('## Lessons Learned');
+  lines.push('- type: lesson');
+  lines.push('  text: <short title>');
+  lines.push('  problem: <what went wrong>');
+  lines.push('  resolution: <what fixed it>');
+  lines.push('');
+  lines.push('## Gaps Filled');
+  lines.push('- type: decision|pattern|mistake|note');
+  lines.push('  text: <new entry to add>');
+  lines.push('');
+  lines.push('## Themes');
+  lines.push('- <overarching theme>');
+  lines.push('');
+  lines.push('## Summary');
+  lines.push('<2-3 sentence summary>');
+
+  return { empty: false, prompt: lines.join('\n'), state };
+}