brainclaw 0.29.2 → 1.5.4

package/dist/core/memory-compactor.js

@@ -0,0 +1,432 @@
+/**
+ * Memory compactor — semantic consolidation of near-duplicate and stale memory items.
+ *
+ * Scans constraints, decisions, and traps for similarity clusters using the Dice
+ * coefficient, scores freshness, and proposes merges or archival.
+ *
+ * Three consumption modes:
+ *   1. `analyzeMemory()` → dry-run report (clusters + stale items)
+ *   2. `applyCompaction()` → execute merges, archive originals
+ *   3. `suggestCompaction()` → lightweight hint for session_end
+ *
+ * Non-destructive: merged items are archived to JSONL, never deleted outright.
+ *
+ * @module
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { normalise, similarity } from './duplicates.js';
+import { resolveEntityDir } from './io.js';
+import { loadState, saveState } from './state.js';
+import { mutate } from './mutation-pipeline.js';
+import { logger } from './logger.js';
+const DEFAULT_SIMILARITY_THRESHOLD = 0.55;
+const DEFAULT_STALENESS_MAX_DAYS = 180;
+const DEFAULT_STALENESS_SCORE_THRESHOLD = 0.20;
+const DEFAULT_MIN_CLUSTER_SIZE = 2;
+// ---------------------------------------------------------------------------
+// Analysis
+// ---------------------------------------------------------------------------
+/**
+ * Analyze the full memory state and produce a compaction report.
+ * Pure function — reads state, does not mutate.
+ */
+export function analyzeMemory(state, options = {}) {
+    const threshold = options.similarityThreshold ?? DEFAULT_SIMILARITY_THRESHOLD;
+    const stalenessMaxDays = options.stalenessMaxDays ?? DEFAULT_STALENESS_MAX_DAYS;
+    const stalenessScoreThreshold = options.stalenessScoreThreshold ?? DEFAULT_STALENESS_SCORE_THRESHOLD;
+    const minClusterSize = options.minClusterSize ?? DEFAULT_MIN_CLUSTER_SIZE;
+    // Flatten all memory items with their type tag
+    const items = flattenMemoryItems(state);
+    const totalItems = items.length;
+    // Build a reference count index (how many times each ID is mentioned across all items)
+    const refCounts = buildReferenceIndex(state);
+    // Cluster by type
+    const clusters = [];
+    for (const type of ['constraint', 'decision', 'trap']) {
+        const typed = items.filter(i => i.type === type);
+        const typeClusters = findClusters(typed, threshold, minClusterSize);
+        clusters.push(...typeClusters);
+    }
+    // Find stale items (not already in a cluster)
+    const clusteredIds = new Set(clusters.flatMap(c => c.items.map(i => i.id)));
+    const staleItems = [];
+    for (const item of items) {
+        if (clusteredIds.has(item.id))
+            continue;
+        const score = computeStalenessScore(item, refCounts, stalenessMaxDays);
+        if (score < stalenessScoreThreshold) {
+            staleItems.push({
+                ...item,
+                score: Math.round(score * 100) / 100,
+                reason: score === 0
+                    ? `older than ${stalenessMaxDays} days with no references`
+                    : `low confidence (freshness × reference density)`,
+            });
+        }
+    }
+    // Sort stale by score ascending (most stale first)
+    staleItems.sort((a, b) => a.score - b.score);
+    const archivableCount = clusters.reduce((n, c) => n + c.items.length - 1, 0) + staleItems.length;
+    return {
+        clusters,
+        staleItems,
+        totalItems,
+        archivableCount,
+        estimatedReductionPct: totalItems > 0 ? Math.round((archivableCount / totalItems) * 100) : 0,
+    };
+}
+/**
+ * Lightweight analysis returning only a short summary string,
+ * suitable for embedding in session_end output.
+ * Returns undefined if nothing actionable.
+ */
+export function suggestCompaction(state, options = {}) {
+    const report = analyzeMemory(state, options);
+    if (report.clusters.length === 0 && report.staleItems.length === 0)
+        return undefined;
+    const parts = [];
+    if (report.clusters.length > 0) {
+        const totalDups = report.clusters.reduce((n, c) => n + c.items.length - 1, 0);
+        parts.push(`${report.clusters.length} similar cluster(s) (${totalDups} mergeable items)`);
+    }
+    if (report.staleItems.length > 0) {
+        parts.push(`${report.staleItems.length} stale item(s)`);
+    }
+    return `Memory compaction opportunity: ${parts.join(', ')}. Run \`brainclaw prune --semantic --dry-run\` for details.`;
+}
+// ---------------------------------------------------------------------------
+// Apply compaction
+// ---------------------------------------------------------------------------
+/**
+ * Analyze and apply compaction atomically under a single mutation lock.
+ * This prevents race conditions where another agent modifies state between
+ * analysis and application.
+ */
+export function analyzeAndApply(options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    let report;
+    let archivedCount = 0;
+    let mergedClusters = 0;
+    let staleArchived = 0;
+    mutate({ cwd }, () => {
+        const state = loadState(cwd);
+        report = analyzeMemory(state, options);
+        if (report.archivableCount === 0)
+            return;
+        const applied = applyReportToState(report, state, cwd);
+        archivedCount = applied.archivedCount;
+        mergedClusters = applied.mergedClusters;
+        staleArchived = applied.staleArchived;
+        saveState(state, cwd);
+    });
+    return { report, result: { archivedCount, mergedClusters, staleArchived } };
+}
+/**
+ * Apply a pre-computed compaction report. Re-validates that items still exist
+ * in the current state before archiving, to handle concurrent modifications.
+ * Runs inside the mutation lock.
+ */
+export function applyCompaction(report, options = {}) {
+    const cwd = options.cwd ?? process.cwd();
+    let archivedCount = 0;
+    let mergedClusters = 0;
+    let staleArchived = 0;
+    mutate({ cwd }, () => {
+        const state = loadState(cwd);
+        const applied = applyReportToState(report, state, cwd);
+        archivedCount = applied.archivedCount;
+        mergedClusters = applied.mergedClusters;
+        staleArchived = applied.staleArchived;
+        saveState(state, cwd);
+    });
+    return { archivedCount, mergedClusters, staleArchived };
+}
+/** Shared logic: apply report mutations to a loaded state. Validates items still exist. */
+function applyReportToState(report, state, cwd) {
+    let archivedCount = 0;
+    let mergedClusters = 0;
+    let staleArchived = 0;
+    for (const cluster of report.clusters) {
+        // Validate keeper still exists in state — skip cluster if not
+        const keeper = findInState(state, cluster.keepId, cluster.type);
+        if (!keeper)
+            continue;
+        // Only archive items that still exist in current state
+        const archiveIds = new Set(cluster.items
+            .filter(i => i.id !== cluster.keepId && findInState(state, i.id, cluster.type))
+            .map(i => i.id));
+        if (archiveIds.size === 0)
+            continue;
+        // Merge tags into keeper
+        const allTags = new Set(keeper.tags);
+        for (const item of cluster.items) {
+            for (const tag of item.tags)
+                allTags.add(tag);
+        }
+        keeper.tags = [...allTags];
+        const entityName = entityNameForType(cluster.type);
+        const archived = archiveItems(archiveIds, entityName, cwd);
+        removeFromState(state, archiveIds, cluster.type);
+        archivedCount += archived;
+        mergedClusters++;
+    }
+    for (const stale of report.staleItems) {
+        // Validate item still exists in current state
+        if (!findInState(state, stale.id, stale.type))
+            continue;
+        const entityName = entityNameForType(stale.type);
+        const archived = archiveItems(new Set([stale.id]), entityName, cwd);
+        removeFromState(state, new Set([stale.id]), stale.type);
+        staleArchived += archived;
+        archivedCount += archived;
+    }
+    return { archivedCount, mergedClusters, staleArchived };
+}
+// ---------------------------------------------------------------------------
+// Clustering (union-find with Dice coefficient)
+// ---------------------------------------------------------------------------
+function findClusters(items, threshold, minSize) {
+    if (items.length < 2)
+        return [];
+    // Normalise texts once
+    const normalised = items.map(item => normalise(item.text));
+    // Union-find structure
+    const parent = items.map((_, i) => i);
+    const find = (i) => {
+        while (parent[i] !== i) {
+            parent[i] = parent[parent[i]];
+            i = parent[i];
+        }
+        return i;
+    };
+    const union = (a, b) => { parent[find(a)] = find(b); };
+    // Pairwise similarity — union items above threshold
+    const pairSim = new Map();
+    for (let i = 0; i < items.length; i++) {
+        for (let j = i + 1; j < items.length; j++) {
+            const sim = similarity(normalised[i], normalised[j]);
+            if (sim >= threshold) {
+                union(i, j);
+                pairSim.set(`${i}:${j}`, sim);
+            }
+        }
+    }
+    // Group by root
+    const groups = new Map();
+    for (let i = 0; i < items.length; i++) {
+        const root = find(i);
+        if (!groups.has(root))
+            groups.set(root, []);
+        groups.get(root).push(i);
+    }
+    // Build clusters from groups >= minSize
+    const clusters = [];
+    for (const indices of groups.values()) {
+        if (indices.length < minSize)
+            continue;
+        const type = items[indices[0]].type;
+        // Compute average pairwise similarity
+        let simSum = 0;
+        let simCount = 0;
+        for (let a = 0; a < indices.length; a++) {
+            for (let b = a + 1; b < indices.length; b++) {
+                const key1 = `${Math.min(indices[a], indices[b])}:${Math.max(indices[a], indices[b])}`;
+                simSum += pairSim.get(key1) ?? similarity(normalised[indices[a]], normalised[indices[b]]);
+                simCount++;
+            }
+        }
+        const avgSimilarity = simCount > 0 ? Math.round((simSum / simCount) * 100) / 100 : 0;
+        // Pick keeper: most recent, then longest text
+        const sorted = [...indices].sort((a, b) => {
+            const dateComp = items[b].created_at.localeCompare(items[a].created_at);
+            if (dateComp !== 0)
+                return dateComp;
+            return items[b].text.length - items[a].text.length;
+        });
+        const keepIdx = sorted[0];
+        const keepId = items[keepIdx].id;
+        const clusterItems = indices.map(idx => ({
+            ...items[idx],
+            similarity: idx === keepIdx ? 1.0 : (pairSim.get(`${Math.min(idx, keepIdx)}:${Math.max(idx, keepIdx)}`)
+                ?? similarity(normalised[idx], normalised[keepIdx])),
+        }));
+        // Sort: keeper first, then by similarity desc
+        clusterItems.sort((a, b) => {
+            if (a.id === keepId)
+                return -1;
+            if (b.id === keepId)
+                return 1;
+            return b.similarity - a.similarity;
+        });
+        clusters.push({ type, items: clusterItems, avgSimilarity, keepId });
+    }
+    // Sort clusters by size descending
+    clusters.sort((a, b) => b.items.length - a.items.length);
+    return clusters;
+}
+// ---------------------------------------------------------------------------
+// Staleness scoring
+// ---------------------------------------------------------------------------
+function computeStalenessScore(item, refCounts, maxDays) {
+    const ageMs = Date.now() - Date.parse(item.created_at);
+    const ageDays = ageMs / (24 * 60 * 60 * 1000);
+    const freshness = Math.max(0, 1 - ageDays / maxDays);
+    const refs = refCounts.get(item.id) ?? 0;
+    // Referenced items get a floor so they're never fully stale —
+    // an actively referenced item is useful regardless of age.
+    const effective = Math.max(freshness, refs > 0 ? 0.25 : 0);
+    return effective * (1 + Math.log1p(refs));
+}
+/**
+ * Build an index of how many times each item ID is referenced across
+ * all plan_items (text + plan_id) and handoffs (text + plan_id + contract).
+ */
+function buildReferenceIndex(state) {
+    const counts = new Map();
+    const increment = (id) => counts.set(id, (counts.get(id) ?? 0) + 1);
+    // Collect all IDs we care about
+    const allIds = new Set();
+    for (const c of state.active_constraints)
+        allIds.add(c.id);
+    for (const d of state.recent_decisions)
+        allIds.add(d.id);
+    for (const t of state.known_traps)
+        allIds.add(t.id);
+    // Scan plans for references (plans reference memory items by ID in their text)
+    for (const plan of state.plan_items) {
+        for (const id of allIds) {
+            if (plan.text.includes(id))
+                increment(id);
+        }
+    }
+    // Scan handoffs for references
+    for (const handoff of state.open_handoffs) {
+        for (const id of allIds) {
+            if (handoff.text.includes(id))
+                increment(id);
+            if (handoff.plan_id === id)
+                increment(id);
+        }
+    }
+    // Cross-references within memory items themselves
+    const allItems = [
+        ...state.active_constraints,
+        ...state.recent_decisions,
+        ...state.known_traps,
+    ];
+    for (const item of allItems) {
+        for (const id of allIds) {
+            if (id !== item.id && item.text.includes(id))
+                increment(id);
+        }
+    }
+    return counts;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function flattenMemoryItems(state) {
+    const items = [];
+    for (const c of state.active_constraints) {
+        items.push({ id: c.id, text: c.text, created_at: c.created_at, tags: c.tags, type: 'constraint', related_paths: c.related_paths });
+    }
+    for (const d of state.recent_decisions) {
+        items.push({ id: d.id, text: d.text, created_at: d.created_at, tags: d.tags, type: 'decision', related_paths: d.related_paths });
+    }
+    for (const t of state.known_traps) {
+        items.push({ id: t.id, text: t.text, created_at: t.created_at, tags: t.tags, type: 'trap', related_paths: t.related_paths });
+    }
+    return items;
+}
+function entityNameForType(type) {
+    switch (type) {
+        case 'constraint': return 'constraints';
+        case 'decision': return 'decisions';
+        case 'trap': return 'traps';
+    }
+}
+function findInState(state, id, type) {
+    switch (type) {
+        case 'constraint': return state.active_constraints.find(c => c.id === id);
+        case 'decision': return state.recent_decisions.find(d => d.id === id);
+        case 'trap': return state.known_traps.find(t => t.id === id);
+    }
+}
+function removeFromState(state, ids, type) {
+    switch (type) {
+        case 'constraint':
+            state.active_constraints = state.active_constraints.filter(c => !ids.has(c.id));
+            break;
+        case 'decision':
+            state.recent_decisions = state.recent_decisions.filter(d => !ids.has(d.id));
+            break;
+        case 'trap':
+            state.known_traps = state.known_traps.filter(t => !ids.has(t.id));
+            break;
+    }
+}
+/**
+ * Archive items by ID: read their JSON files, append to compacted.jsonl, delete originals.
+ * Returns number of items archived.
+ */
+function archiveItems(ids, entityName, cwd) {
+    const dir = resolveEntityDir(entityName, cwd, 'read');
+    const archivePath = path.join(resolveEntityDir(entityName, cwd, 'write'), 'compacted.jsonl');
+    let archived = 0;
+    for (const id of ids) {
+        const filePath = path.join(dir, `${id}.json`);
+        if (!fs.existsSync(filePath))
+            continue;
+        try {
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const item = JSON.parse(content);
+            item._compacted_at = new Date().toISOString();
+            fs.mkdirSync(path.dirname(archivePath), { recursive: true });
+            fs.appendFileSync(archivePath, JSON.stringify(item) + '\n', 'utf-8');
+            fs.unlinkSync(filePath);
+            archived++;
+        }
+        catch (err) {
+            logger.debug(`Failed to archive ${entityName}/${id}:`, err);
+        }
+    }
+    return archived;
+}
+// ---------------------------------------------------------------------------
+// Report formatting (CLI output)
+// ---------------------------------------------------------------------------
+export function formatReport(report) {
+    const lines = [];
+    lines.push(`Memory compaction analysis — ${report.totalItems} items scanned\n`);
+    if (report.clusters.length > 0) {
+        lines.push(`Similar clusters (${report.clusters.length}):`);
+        for (const cluster of report.clusters) {
+            lines.push(`  Cluster (${cluster.items.length} ${cluster.type}s, avg similarity: ${cluster.avgSimilarity}):`);
+            for (const item of cluster.items) {
+                const marker = item.id === cluster.keepId ? 'KEEP' : 'archive';
+                const preview = item.text.length > 80 ? item.text.slice(0, 77) + '...' : item.text;
+                lines.push(`    [${marker}] ${item.id} (${item.created_at.slice(0, 10)}) ${preview}`);
+            }
+        }
+        lines.push('');
+    }
+    if (report.staleItems.length > 0) {
+        lines.push(`Stale items (${report.staleItems.length}):`);
+        for (const item of report.staleItems) {
+            const preview = item.text.length > 80 ? item.text.slice(0, 77) + '...' : item.text;
+            lines.push(`  [${item.type}] ${item.id} (score: ${item.score}) ${preview}`);
+            lines.push(`    → ${item.reason}`);
+        }
+        lines.push('');
+    }
+    if (report.archivableCount > 0) {
+        lines.push(`Estimated reduction: ${report.archivableCount}/${report.totalItems} items (${report.estimatedReductionPct}%)`);
+        lines.push(`Run \`brainclaw prune --semantic\` to apply.`);
+    }
+    else {
+        lines.push('No compaction opportunities found.');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=memory-compactor.js.map

package/dist/core/memory-git.js

@@ -4,6 +4,44 @@ import path from 'node:path';
 import { memoryDir } from './io.js';
 import { logger } from './logger.js';
 const GIT_DIR_NAME = '.git';
+const ROLLBACK_ROOT_FILES = new Set([
+    'config.yaml',
+    'state.yaml',
+    'project.md',
+    'project.identity.json',
+    '.id-counter.json',
+]);
+const ROLLBACK_EXCLUDED_BASENAMES = new Set([
+    'archive.jsonl',
+    'compacted.jsonl',
+]);
+const ROLLBACK_ALLOWED_EXTENSIONS = new Set([
+    '.json',
+    '.yaml',
+    '.yml',
+]);
+const ROLLBACK_ROOTS = [
+    'constraints/',
+    'decisions/',
+    'traps/',
+    'instructions/',
+    'plans/',
+    'sequences/',
+    'claims/',
+    'handoffs/',
+    'surface-tasks/',
+    'memory/constraints/',
+    'memory/decisions/',
+    'memory/traps/',
+    'memory/instructions/',
+    'coordination/plans/',
+    'coordination/sequences/',
+    'coordination/claims/',
+    'coordination/handoffs/',
+    'coordination/sessions/',
+    'coordination/surface-tasks/',
+    'sessions/',
+];
 /**
  * Check if the memory directory has an internal git repo.
  */
@@ -87,7 +125,13 @@ export function getMemoryLog(limit = 20, cwd) {
     }
 }
 /**
- * Rollback the memory to a previous commit.
+ * Restore live files from the current project's Brainclaw store to a previous
+ * commit without deleting durable logs, archives, or compaction outputs.
+ *
+ * Behaviour note: this is intentionally selective and non-destructive.
+ * It replaces an older "full-store reset" rollback model.
+ *
+ * This intentionally creates a new commit instead of performing a hard reset.
  * Returns true if successful.
  */
 export function rollbackMemory(ref, cwd) {
@@ -95,13 +139,27 @@ export function rollbackMemory(ref, cwd) {
         return false;
     const dir = memoryDir(cwd);
     try {
-        // Remove all tracked files, then restore from the target ref
-        // This ensures files added after the ref are also removed
-        git(dir, ['rm', '-rf', '--quiet', '--ignore-unmatch', '.']);
-        git(dir, ['checkout', ref, '--', '.']);
-        git(dir, ['clean', '-fd', '--quiet']);
-        git(dir, ['add', '-A']);
-        git(dir, ['commit', '--quiet', '--allow-empty', '-m', `brainclaw: rollback to ${ref}`]);
+        git(dir, ['rev-parse', '--verify', `${ref}^{commit}`]);
+        const currentLiveFiles = listRollbackManagedFilesOnDisk(dir);
+        const targetLiveFiles = listRollbackManagedFilesAtRef(dir, ref);
+        const targetSet = new Set(targetLiveFiles);
+        for (const relPath of currentLiveFiles) {
+            if (!targetSet.has(relPath)) {
+                removeManagedPath(dir, relPath);
+            }
+        }
+        for (const relPath of targetLiveFiles) {
+            restoreManagedPathFromRef(dir, ref, relPath);
+        }
+        stageManagedPaths(dir, new Set([...currentLiveFiles, ...targetLiveFiles]));
+        const staged = git(dir, ['diff', '--cached', '--name-only'])
+            .trim()
+            .split('\n')
+            .map((entry) => normalizeRelativePath(entry))
+            .filter((entry) => entry.length > 0 && isRollbackManagedPath(entry));
+        if (staged.length === 0)
+            return false;
+        git(dir, ['commit', '--quiet', '-m', `brainclaw: rollback live memory to ${ref}`]);
         return true;
     }
     catch (err) {
@@ -130,4 +188,90 @@ function git(cwd, args) {
         stdio: ['pipe', 'pipe', 'pipe'],
     });
 }
+function normalizeRelativePath(filepath) {
+    return filepath.replace(/\\/g, '/').replace(/^\.\/+/, '').trim();
+}
+function isRollbackManagedPath(relativePath) {
+    const normalized = normalizeRelativePath(relativePath);
+    if (!normalized)
+        return false;
+    if (ROLLBACK_ROOT_FILES.has(normalized))
+        return true;
+    const rootMatched = ROLLBACK_ROOTS.some((prefix) => normalized.startsWith(prefix));
+    if (!rootMatched)
+        return false;
+    const basename = path.posix.basename(normalized);
+    if (ROLLBACK_EXCLUDED_BASENAMES.has(basename))
+        return false;
+    const extension = path.posix.extname(basename).toLowerCase();
+    return ROLLBACK_ALLOWED_EXTENSIONS.has(extension);
+}
+function listRollbackManagedFilesAtRef(cwd, ref) {
+    const output = git(cwd, ['ls-tree', '-r', '--name-only', ref]);
+    return output
+        .split('\n')
+        .map((entry) => normalizeRelativePath(entry))
+        .filter((entry) => entry.length > 0 && isRollbackManagedPath(entry))
+        .sort();
+}
+function listRollbackManagedFilesOnDisk(cwd) {
+    const results = [];
+    walkFiles(cwd, cwd, results);
+    results.sort();
+    return results;
+}
+function walkFiles(baseDir, currentDir, results) {
+    for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
+        if (entry.name === GIT_DIR_NAME)
+            continue;
+        const absolutePath = path.join(currentDir, entry.name);
+        if (entry.isDirectory()) {
+            walkFiles(baseDir, absolutePath, results);
+            continue;
+        }
+        const relativePath = normalizeRelativePath(path.relative(baseDir, absolutePath));
+        if (isRollbackManagedPath(relativePath)) {
+            results.push(relativePath);
+        }
+    }
+}
+function removeManagedPath(cwd, relativePath) {
+    const absolutePath = path.join(cwd, relativePath);
+    if (!fs.existsSync(absolutePath))
+        return;
+    fs.unlinkSync(absolutePath);
+    removeEmptyParentDirs(path.dirname(absolutePath), cwd);
+}
+function removeEmptyParentDirs(startDir, stopDir) {
+    let current = startDir;
+    const resolvedStop = path.resolve(stopDir);
+    while (path.resolve(current).startsWith(resolvedStop) && path.resolve(current) !== resolvedStop) {
+        const entries = fs.readdirSync(current);
+        if (entries.length > 0)
+            return;
+        fs.rmdirSync(current);
+        current = path.dirname(current);
+    }
+}
+function restoreManagedPathFromRef(cwd, ref, relativePath) {
+    const absolutePath = path.join(cwd, relativePath);
+    fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+    const content = gitBuffer(cwd, ['show', `${ref}:${relativePath}`]);
+    fs.writeFileSync(absolutePath, content);
+}
+function stageManagedPaths(cwd, managedPaths) {
+    const paths = [...managedPaths].filter(Boolean);
+    const chunkSize = 64;
+    for (let index = 0; index < paths.length; index += chunkSize) {
+        const chunk = paths.slice(index, index + chunkSize);
+        git(cwd, ['add', '-A', '--', ...chunk]);
+    }
+}
+function gitBuffer(cwd, args) {
+    return execFileSync('git', args, {
+        cwd,
+        timeout: 10_000,
+        stdio: ['pipe', 'pipe', 'pipe'],
+    });
+}
 //# sourceMappingURL=memory-git.js.map