@mrxkun/mcfast-mcp 4.1.15 → 4.2.0

package/src/memory/memory-engine.js

@@ -25,6 +25,8 @@ import { CuratedMemory } from './layers/curated-memory.js';
 
 // Bootstrap
 import { AgentsMdBootstrap } from './bootstrap/agents-md.js';
+import { ProjectScanner } from './bootstrap/project-scanner.js';
+import { MemoryCompactor } from './utils/compaction.js';
 
 // Project Analysis (AI-powered)
 import { execute as projectAnalyzeExecute } from '../tools/project_analyze.js';
@@ -260,6 +262,14 @@ export class MemoryEngine {
     // Initialize intelligence
     await this.initializeIntelligence();
 
+    // Initialize compactor
+    this.compactor = new MemoryCompactor({
+      memoryPath: this.memoryPath,
+      softThresholdKB: 500,
+      hardThresholdKB: 1024,
+      keepDays: 7
+    });
+
     this.isInitialized = true;
 
     console.error(`[MemoryEngine] ✅ Initialized successfully`);
@@ -267,8 +277,18 @@ export class MemoryEngine {
     console.error(`[MemoryEngine] Smart Routing: ${this.smartRoutingEnabled ? 'ENABLED' : 'DISABLED'}`);
     console.error(`[MemoryEngine] Intelligence: ${this.intelligenceEnabled ? 'ENABLED' : 'DISABLED'}`);
 
-    // Auto-analyze project if MCFAST_TOKEN is set
-    await this.autoAnalyzeProject();
+    // Bootstrap: local scan FIRST (no internet needed), AI scan if token present
+    this.bootstrapProjectContext().catch(err => {
+      console.error('[MemoryEngine] Bootstrap error (non-fatal):', err.message);
+    });
+
+    // Periodic compaction (every 30 min, non-blocking)
+    const _compactTimer = setInterval(() => {
+      this.compactor.maybeCompact().then(r => {
+        if (r.ran) console.error('[MemoryEngine] Auto-compaction ran, compacted:', r.compacted);
+      }).catch(() => { });
+    }, 30 * 60 * 1000);
+    if (_compactTimer.unref) _compactTimer.unref();
 
     // Log initialization to daily logs
     await this.dailyLogs.log('Memory Engine Initialized', `Project: ${projectPath}`, {
@@ -276,6 +296,107 @@ export class MemoryEngine {
     });
   }
 
+  /**
+   * Bootstrap project context based on bootstrap_mode setting.
+   * Modes:
+   * - local: only runs ProjectScanner (no internet needed)
+   * - cloud: only runs projectAnalyzeExecute (via Mercury API)
+   * - hybrid: runs ProjectScanner, then enriches via AI
+   */
+  async bootstrapProjectContext() {
+    // Determine bootstrap mode
+    let mode = process.env.MCFAST_BOOTSTRAP_MODE || 'hybrid';
+    if (this.dashboardClient && this.dashboardClient.apiKey) {
+      try {
+        const config = await this.dashboardClient.getSearchConfig();
+        if (config && config.bootstrap_mode) {
+          mode = config.bootstrap_mode;
+        }
+      } catch (e) {
+        console.error('[MemoryEngine] Failed to fetch bootstrap mode, defaulting to hybrid');
+      }
+    }
+
+    // Skip if already has curated (non-auto-generated) context
+    try {
+      const memContent = await this.curatedMemory.read();
+      if (memContent &&
+        memContent.includes('## Project Context') &&
+        !memContent.includes('<!-- Add project context here -->') &&
+        !memContent.includes('Auto-generated by mcfast local scanner')) {
+        console.error('[MemoryEngine] Project Context already curated, skipping auto-scan');
+        return;
+      }
+    } catch { }
+
+    // Step 1: Local Scan (for 'local' or 'hybrid' modes)
+    if (mode === 'local' || mode === 'hybrid') {
+      try {
+        console.error(`[MemoryEngine] 🔍 Local project scan starting... (mode: ${mode})`);
+        const scanner = new ProjectScanner(this.projectPath);
+        const scanResult = await scanner.scan();
+        const contextSection = scanner.buildMemorySection(scanResult);
+        await this._updateMemorySection('Project Context', contextSection);
+        console.error(`[MemoryEngine] ✅ Local scan: ${scanResult.name} (${scanResult.type})`);
+        await this.dailyLogs.log(
+          'Project Scanned',
+          `${scanResult.name} | ${scanResult.type} | routes: ${scanResult.apiRoutes.length}`,
+          { name: scanResult.name, type: scanResult.type }
+        );
+      } catch (err) {
+        console.error('[MemoryEngine] Local scan error:', err.message);
+      }
+    }
+
+    // Step 2: AI enrichment (background, optional, for 'cloud' or 'hybrid' modes)
+    if ((mode === 'cloud' || mode === 'hybrid') && process.env.MCFAST_TOKEN) {
+      console.error(`[MemoryEngine] 🔄 AI analysis starting in background... (mode: ${mode})`);
+      projectAnalyzeExecute({ force: false, updateMemory: true })
+        .then(r => { if (r?.metadata?.memoryUpdated) console.error('[MemoryEngine] ✅ AI analysis complete'); })
+        .catch(err => console.error('[MemoryEngine] ⚠️ AI analysis failed:', err.message));
+    } else if (mode === 'cloud' && !process.env.MCFAST_TOKEN) {
+      console.error('[MemoryEngine] ⚠️ Cloud bootstrap mode requires MCFAST_TOKEN, but none was provided.');
+    }
+  }
+
+  /**
+   * Update or insert a named ## section in MEMORY.md
+   */
+  async _updateMemorySection(sectionName, newContent) {
+    let content = '';
+    try { content = await this.curatedMemory.read() || ''; } catch { }
+    if (!content) {
+      content = `# Long-term Memory\n\n---\n*Last updated: ${new Date().toISOString()}*\n`;
+    }
+    const re = new RegExp(`(## ${sectionName}\n)([\\s\\S]*?)(?=\n## |$)`, 's');
+    const replacement = `$1\n${newContent}\n\n`;
+    if (re.test(content)) {
+      content = content.replace(re, replacement);
+    } else {
+      const footerIdx = content.lastIndexOf('---');
+      const insertAt = footerIdx > 0 ? footerIdx : content.length;
+      content = content.slice(0, insertAt) +
+        `\n## ${sectionName}\n\n${newContent}\n\n` +
+        content.slice(insertAt);
+    }
+    content = content.replace(/\*Last updated:.*\*/, `*Last updated: ${new Date().toISOString()}*`);
+    const memFile = path.join(this.memoryPath, 'MEMORY.md');
+    await fs.mkdir(path.dirname(memFile), { recursive: true });
+    await fs.writeFile(memFile, content, 'utf-8');
+  }
+
+  /** Get compaction statistics */
+  async getCompactionStats() {
+    if (!this.compactor) return null;
+    return this.compactor.getStats();
+  }
+
+  /** Manually trigger memory compaction */
+  async triggerCompaction(force = false) {
+    if (!this.compactor) throw new Error('Compactor not initialized');
+    return force ? this.compactor.forceCompact() : this.compactor.maybeCompact();
+  }
+
   async initializeIntelligence() {
     if (!this.intelligenceEnabled) return;
 

package/src/memory/utils/chunker.js

@@ -29,7 +29,7 @@ export class Chunker {
 
       if (currentTokens + lineTokens > this.chunkSize && currentChunk.length > 0) {
         chunks.push(this.createChunk(currentChunk, chunkStartLine, i, metadata));
-        
+
         const overlapLines = this.calculateOverlap(currentChunk);
         currentChunk = [...overlapLines, line];
         currentTokens = overlapLines.reduce((sum, l) => sum + this.estimateTokens(l), 0) + lineTokens;
@@ -54,7 +54,7 @@ export class Chunker {
     for (let i = lines.length - 1; i >= 0; i--) {
       const line = lines[i];
       const lineTokens = this.estimateTokens(line);
-      
+
       if (overlapTokens + lineTokens <= this.overlap) {
         overlapLines.unshift(line);
         overlapTokens += lineTokens;
@@ -69,7 +69,7 @@ export class Chunker {
   createChunk(lines, startLine, endLine, metadata) {
     const content = lines.join('\n');
     const tokens = this.estimateTokens(content);
-    
+
     // Handle both string (filePath) and object metadata
     const filePath = typeof metadata === 'string' ? metadata : (metadata.filePath || '');
 
@@ -80,7 +80,8 @@ export class Chunker {
       content_hash: crypto.createHash('md5').update(content).digest('hex'),
       start_line: startLine,
       end_line: endLine,
-      token_count: tokens
+      token_count: tokens,
+      chunk_type: (typeof metadata === 'object' && metadata.chunkType) || 'code'
     };
   }
 

package/src/memory/utils/compaction.js

@@ -0,0 +1,281 @@
+/**
+ * Memory Compaction Engine
+ *
+ * Inspired by OpenClaw's "automatic memory flush (pre-compaction ping)" design.
+ * https://docs.openclaw.ai/concepts/memory#automatic-memory-flush-pre-compaction-ping
+ *
+ * Strategy:
+ *   1. Monitor MEMORY.md + daily logs total size
+ *   2. When total > softThresholdKB: compact old daily logs (>= keepDays)
+ *      into MEMORY.md "Archived Sessions" section, then delete them
+ *   3. When MEMORY.md itself > hardThresholdKB: trim old "Archived Sessions"
+ *      entries (keep only the compressed summary lines)
+ *   4. Run silently in background; report stats on request
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+
+export class MemoryCompactor {
+    constructor(options = {}) {
+        this.memoryPath = options.memoryPath; // e.g. .mcfast/
+        this.softThresholdKB = options.softThresholdKB || 500;   // compact old logs
+        this.hardThresholdKB = options.hardThresholdKB || 1024;  // trim MEMORY.md
+        this.keepDays = options.keepDays || 7;                    // keep last N daily logs
+        this.archiveSectionName = 'Archived Sessions';
+    }
+
+    get memoryDir() { return path.join(this.memoryPath, 'memory'); }
+    get curatedFile() { return path.join(this.memoryPath, 'MEMORY.md'); }
+
+    /**
+     * Check if compaction is needed and run it if so.
+     * Returns an object with what was done.
+     */
+    async maybeCompact() {
+        const totalKB = await this.getTotalSizeKB();
+        const report = { totalKB, softThreshold: this.softThresholdKB, ran: false };
+
+        if (totalKB < this.softThresholdKB) {
+            return report;
+        }
+
+        report.ran = true;
+        report.compacted = await this.compactOldLogs();
+
+        // If MEMORY.md still huge, trim archive section
+        const curatedKB = await this.getFileSizeKB(this.curatedFile);
+        if (curatedKB > this.hardThresholdKB) {
+            report.trimmed = await this.trimArchivedSessions();
+        }
+
+        return report;
+    }
+
+    /**
+     * Force compaction regardless of threshold.
+     */
+    async forceCompact() {
+        const compacted = await this.compactOldLogs();
+        const trimmed = await this.trimArchivedSessions();
+        return { compacted, trimmed };
+    }
+
+    /**
+     * Get total size (in KB) of MEMORY.md + all daily logs
+     */
+    async getTotalSizeKB() {
+        let total = 0;
+        total += await this.getFileSizeKB(this.curatedFile);
+
+        try {
+            const files = await fs.readdir(this.memoryDir);
+            for (const f of files) {
+                if (f.endsWith('.md')) {
+                    total += await this.getFileSizeKB(path.join(this.memoryDir, f));
+                }
+            }
+        } catch { }
+
+        return total;
+    }
+
+    async getFileSizeKB(filePath) {
+        try {
+            const stat = await fs.stat(filePath);
+            return stat.size / 1024;
+        } catch {
+            return 0;
+        }
+    }
+
+    /**
+     * Compact daily logs older than keepDays into MEMORY.md archive section,
+     * then delete them.
+     * Returns number of files compacted.
+     */
+    async compactOldLogs() {
+        let compacted = 0;
+        const cutoffDate = new Date();
+        cutoffDate.setDate(cutoffDate.getDate() - this.keepDays);
+
+        let files = [];
+        try {
+            files = await fs.readdir(this.memoryDir);
+        } catch {
+            return 0;
+        }
+
+        const oldLogs = files
+            .filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+            .filter(f => {
+                const fileDate = new Date(f.replace('.md', ''));
+                return fileDate < cutoffDate;
+            })
+            .sort();
+
+        if (oldLogs.length === 0) return 0;
+
+        // Read all old logs and build archive entries
+        const archiveLines = [];
+        for (const logFile of oldLogs) {
+            const filePath = path.join(this.memoryDir, logFile);
+            try {
+                const content = await fs.readFile(filePath, 'utf-8');
+                const dateStr = logFile.replace('.md', '');
+                const summary = this.summarizeLog(content, dateStr);
+                archiveLines.push(summary);
+                await fs.unlink(filePath);
+                compacted++;
+            } catch (err) {
+                console.error(`[MemoryCompactor] Failed to compact ${logFile}:`, err.message);
+            }
+        }
+
+        // Append to MEMORY.md archive section
+        if (archiveLines.length > 0) {
+            await this.appendToArchiveSection(archiveLines.join('\n\n'));
+        }
+
+        console.error(`[MemoryCompactor] Compacted ${compacted} old daily logs into ${this.archiveSectionName}`);
+        return compacted;
+    }
+
+    /**
+     * Summarize a daily log into a compact bullet list
+     */
+    summarizeLog(content, dateStr) {
+        const lines = content.split('\n').filter(l => l.trim());
+        // Extract headers as key events
+        const events = lines
+            .filter(l => l.startsWith('## ') || l.startsWith('### '))
+            .map(l => l.replace(/^#+\s*/, '').trim())
+            .slice(0, 5);
+
+        if (events.length === 0) {
+            // Fallback: take first meaningful lines
+            const nonEmpty = lines.filter(l => l.length > 10 && !l.startsWith('#')).slice(0, 3);
+            events.push(...nonEmpty.map(l => l.substring(0, 120)));
+        }
+
+        const bulletList = events.map(e => `  - ${e}`).join('\n');
+        return `### ${dateStr}\n${bulletList}`;
+    }
+
+    /**
+     * Append content to the "Archived Sessions" section in MEMORY.md
+     */
+    async appendToArchiveSection(newContent) {
+        let content = '';
+        try {
+            content = await fs.readFile(this.curatedFile, 'utf-8');
+        } catch {
+            content = this.getDefaultTemplate();
+        }
+
+        const sectionHeader = `## ${this.archiveSectionName}`;
+        if (content.includes(sectionHeader)) {
+            // Append inside existing section
+            content = content.replace(
+                new RegExp(`(## ${this.archiveSectionName}[\\s\\S]*?)(?=\\n## |$)`, 's'),
+                (match) => match.trimEnd() + '\n\n' + newContent + '\n'
+            );
+        } else {
+            // Add section before the footer
+            content = content.replace(
+                /(---\n\*Last updated:.*\*)/,
+                `\n${sectionHeader}\n\n${newContent}\n\n$1`
+            );
+            if (!content.includes(sectionHeader)) {
+                content += `\n\n${sectionHeader}\n\n${newContent}\n`;
+            }
+        }
+
+        // Update timestamp
+        content = content.replace(
+            /\*Last updated:.*\*/,
+            `*Last updated: ${new Date().toISOString()}*`
+        );
+
+        await fs.mkdir(path.dirname(this.curatedFile), { recursive: true });
+        await fs.writeFile(this.curatedFile, content);
+    }
+
+    /**
+     * Trim old archive entries when MEMORY.md is too large.
+     * Keeps the last 30 entries in "Archived Sessions", discards older ones.
+     */
+    async trimArchivedSessions() {
+        let content = '';
+        try {
+            content = await fs.readFile(this.curatedFile, 'utf-8');
+        } catch {
+            return 0;
+        }
+
+        const sectionHeader = `## ${this.archiveSectionName}`;
+        const sectionMatch = content.match(
+            new RegExp(`## ${this.archiveSectionName}([\\s\\S]*?)(?=\\n## |$)`, 's')
+        );
+
+        if (!sectionMatch) return 0;
+
+        const sectionBody = sectionMatch[1];
+        // Split into individual entries (each starts with ### YYYY-MM-DD)
+        const entries = sectionBody.split(/(?=###\s+\d{4}-\d{2}-\d{2})/g).filter(e => e.trim());
+
+        if (entries.length <= 30) return 0; // no need to trim
+
+        const KEEP = 30;
+        const trimmed = entries.length - KEEP;
+        const kept = entries.slice(-KEEP);
+
+        const newSection = `${sectionHeader}\n\n<!-- ${trimmed} older entries trimmed for size -->\n\n${kept.join('\n\n')}`;
+        content = content.replace(
+            new RegExp(`## ${this.archiveSectionName}[\\s\\S]*?(?=\\n## |$)`, 's'),
+            newSection
+        );
+
+        await fs.writeFile(this.curatedFile, content);
+        console.error(`[MemoryCompactor] Trimmed ${trimmed} old archive entries`);
+        return trimmed;
+    }
+
+    /**
+     * Get compaction stats without running compaction
+     */
+    async getStats() {
+        const totalKB = await this.getTotalSizeKB();
+        const curatedKB = await this.getFileSizeKB(this.curatedFile);
+
+        let dailyLogsCount = 0;
+        let oldLogsCount = 0;
+        const cutoffDate = new Date();
+        cutoffDate.setDate(cutoffDate.getDate() - this.keepDays);
+
+        try {
+            const files = await fs.readdir(this.memoryDir);
+            const logs = files.filter(f => /^\d{4}-\d{2}-\d{2}\.md$/.test(f));
+            dailyLogsCount = logs.length;
+            oldLogsCount = logs.filter(f => new Date(f.replace('.md', '')) < cutoffDate).length;
+        } catch { }
+
+        return {
+            totalKB: Math.round(totalKB * 10) / 10,
+            curatedKB: Math.round(curatedKB * 10) / 10,
+            dailyLogsCount,
+            oldLogsCount,
+            softThresholdKB: this.softThresholdKB,
+            hardThresholdKB: this.hardThresholdKB,
+            keepDays: this.keepDays,
+            needsCompaction: totalKB >= this.softThresholdKB
+        };
+    }
+
+    getDefaultTemplate() {
+        return `# Long-term Memory\n\n---\n*Last updated: ${new Date().toISOString()}*\n`;
+    }
+}
+
+export default MemoryCompactor;