cf-memory-mcp 3.3.0 → 3.4.0

package/bin/cf-memory-mcp.js

@@ -17,12 +17,15 @@ const https = require('https');
 const { URL } = require('url');
 const os = require('os');
 const process = require('process');
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
 
 // Configuration
-const STREAMABLE_HTTP_URL = 'https://cf-memory-mcp.johnlam90.workers.dev/mcp';
-const LEGACY_SERVER_URL = 'https://cf-memory-mcp.johnlam90.workers.dev/mcp/message';
+const STREAMABLE_HTTP_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
+const LEGACY_SERVER_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
 const PACKAGE_VERSION = require('../package.json').version;
-const TIMEOUT_MS = 30000;
+const TIMEOUT_MS = 60000; // Increased timeout for batch operations
 const CONNECT_TIMEOUT_MS = 10000;
 
 // Get API key from environment variable (will be checked later)
@@ -56,7 +59,7 @@ class CFMemoryMCP {
         this.logDebug(`Legacy Server URL: ${this.legacyServerUrl}`);
         this.logDebug(`User Agent: ${this.userAgent}`);
     }
-    
+
     /**
      * Log debug messages to stderr (won't interfere with MCP communication)
      */
@@ -65,7 +68,7 @@ class CFMemoryMCP {
             process.stderr.write(`[DEBUG] ${new Date().toISOString()} ${message}\n`);
         }
     }
-    
+
     /**
      * Log error messages to stderr
      */
@@ -76,7 +79,7 @@ class CFMemoryMCP {
             process.stderr.write(`[ERROR] ${timestamp} ${error.stack}\n`);
         }
     }
-    
+
     /**
      * Start listening for MCP messages on stdin
      */
@@ -90,7 +93,7 @@ class CFMemoryMCP {
             process.exit(1);
         }
     }
-    
+
     /**
      * Test connectivity to the Cloudflare Worker
      */
@@ -133,37 +136,37 @@ class CFMemoryMCP {
             }
         }
     }
-    
+
     /**
      * Process stdio input/output for MCP communication
      */
     async processStdio() {
         let buffer = '';
-        
+
         // Handle stdin data
         for await (const chunk of process.stdin) {
             buffer += chunk;
-            
+
             // Process complete JSON-RPC messages (one per line)
             let newlineIndex;
             while ((newlineIndex = buffer.indexOf('\n')) !== -1) {
                 const line = buffer.slice(0, newlineIndex).trim();
                 buffer = buffer.slice(newlineIndex + 1);
-                
+
                 if (line) {
                     await this.handleMessage(line);
                 }
             }
         }
-        
+
         // Process any remaining buffer content
         if (buffer.trim()) {
             await this.handleMessage(buffer.trim());
         }
-        
+
         this.logDebug('Stdin closed, shutting down...');
     }
-    
+
     /**
      * Handle a single MCP message
      */
@@ -171,15 +174,46 @@ class CFMemoryMCP {
         try {
             const message = JSON.parse(messageStr);
             this.logDebug(`Processing message: ${message.method} (id: ${message.id})`);
-            
+
+            // Handle lifecycle methods locally
+            if (message.method === 'initialize') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: {
+                        protocolVersion: '2025-03-26',
+                        capabilities: {
+                            tools: {}
+                        },
+                        serverInfo: {
+                            name: 'cf-memory-mcp',
+                            version: PACKAGE_VERSION
+                        }
+                    }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            if (message.method === 'notifications/initialized') {
+                this.logDebug('MCP handshake completed');
+                return;
+            }
+
+            // Intercept index_project tool call to perform local scanning
+            if (message.method === 'tools/call' && message.params && message.params.name === 'index_project') {
+                await this.handleIndexProject(message);
+                return;
+            }
+
             const response = await this.makeRequest(message);
-            
+
             // Send response to stdout
             process.stdout.write(JSON.stringify(response) + '\n');
-            
+
         } catch (error) {
             this.logError('Error handling message:', error);
-            
+
             // Send error response
             const errorResponse = {
                 jsonrpc: '2.0',
@@ -193,16 +227,236 @@ class CFMemoryMCP {
             process.stdout.write(JSON.stringify(errorResponse) + '\n');
         }
     }
-    
+
+    /**
+     * Handle local project indexing - scans local files and sends them via MCP
+     */
+    async handleIndexProject(message) {
+        const { project_path, project_name, include_patterns, exclude_patterns } = message.params.arguments;
+        const resolvedPath = path.resolve(project_path);
+        const name = project_name || path.basename(resolvedPath);
+
+        this.logDebug(`Intercepted index_project for: ${resolvedPath} (${name})`);
+
+        try {
+            // 1. Scan Local Files
+            this.logDebug(`Scanning files in ${resolvedPath}...`);
+            const files = this.scanDirectory(resolvedPath, '', include_patterns, exclude_patterns);
+            this.logDebug(`Found ${files.length} files to index.`);
+
+            if (files.length === 0) {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: {
+                        content: [{
+                            type: 'text',
+                            text: JSON.stringify({
+                                project_id: null,
+                                project_name: name,
+                                files_found: 0,
+                                files_indexed: 0,
+                                chunks_created: 0,
+                                status: 'complete',
+                                message: 'No matching files found in directory'
+                            }, null, 2)
+                        }]
+                    }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // 2. Send files directly via MCP tool call (batched to avoid timeouts)
+            const BATCH_SIZE = 20; // Process 20 files at a time
+            let totalIndexed = 0;
+            let totalChunks = 0;
+            let projectId = null;
+
+            for (let i = 0; i < files.length; i += BATCH_SIZE) {
+                const batch = files.slice(i, i + BATCH_SIZE);
+                this.logDebug(`Processing batch ${Math.floor(i/BATCH_SIZE) + 1}/${Math.ceil(files.length/BATCH_SIZE)} (${batch.length} files)`);
+
+                const projectResult = await this.makeRequest({
+                    jsonrpc: '2.0',
+                    id: `index-batch-${Date.now()}-${i}`,
+                    method: 'tools/call',
+                    params: {
+                        name: 'index_project',
+                        arguments: {
+                            project_path: resolvedPath,
+                            project_name: name,
+                            files: batch.map(f => ({
+                                path: f.relativePath,
+                                content: f.content
+                            })),
+                            include_patterns,
+                            exclude_patterns
+                        }
+                    }
+                });
+
+                if (projectResult.error) {
+                    this.logError(`Batch ${i} failed:`, projectResult.error);
+                    continue;
+                }
+
+                try {
+                    const content = JSON.parse(projectResult.result.content[0].text);
+                    projectId = content.project_id;
+                    totalIndexed += content.files_indexed || 0;
+                    totalChunks += content.chunks_created || 0;
+                } catch (e) {
+                    this.logError('Could not parse batch result:', e);
+                }
+            }
+
+            // 3. Return aggregated success
+            const response = {
+                jsonrpc: '2.0',
+                id: message.id,
+                result: {
+                    content: [{
+                        type: 'text',
+                        text: JSON.stringify({
+                            project_id: projectId,
+                            project_name: name,
+                            files_found: files.length,
+                            files_indexed: totalIndexed,
+                            chunks_created: totalChunks,
+                            status: 'complete'
+                        }, null, 2)
+                    }]
+                }
+            };
+            process.stdout.write(JSON.stringify(response) + '\n');
+
+        } catch (error) {
+            this.logError('Index project failed:', error);
+            const response = {
+                jsonrpc: '2.0',
+                id: message.id,
+                error: {
+                    code: -32000,
+                    message: `Indexing failed: ${error.message}`
+                }
+            };
+            process.stdout.write(JSON.stringify(response) + '\n');
+        }
+    }
+
     /**
-     * Make HTTP request to the Cloudflare Worker
+     * Recursive directory scan with filtering
+     */
+    scanDirectory(dir, basePath = '', includePatterns = null, excludePatterns = null) {
+        let files = [];
+
+        // Default exclusions (always excluded)
+        const DEFAULT_EXCLUDE_DIRS = [
+            'node_modules', '.git', '.svn', '.hg',
+            'dist', 'build', 'out', '.next', '.nuxt',
+            '__pycache__', '.pytest_cache', 'venv', '.venv',
+            'coverage', '.coverage', '.nyc_output',
+            '.wrangler', '.turbo', '.cache'
+        ];
+
+        // Default file extensions to include
+        const DEFAULT_INCLUDE_EXTS = [
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.rb', '.go', '.rs', '.java', '.kt', '.scala',
+            '.cs', '.cpp', '.c', '.h', '.hpp', '.swift', '.php',
+            '.sql', '.sh', '.bash',
+            '.json', '.yaml', '.yml', '.toml',
+            '.html', '.css', '.scss', '.vue', '.svelte',
+            '.md', '.mdx'
+        ];
+
+        // Merge custom exclude patterns
+        const effectiveExcludes = excludePatterns
+            ? [...DEFAULT_EXCLUDE_DIRS, ...excludePatterns]
+            : DEFAULT_EXCLUDE_DIRS;
+
+        try {
+            const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+            for (const entry of entries) {
+                const fullPath = path.join(dir, entry.name);
+                const relPath = path.join(basePath, entry.name);
+
+                if (entry.isDirectory()) {
+                    // Check if directory should be excluded
+                    const shouldExclude = effectiveExcludes.some(pattern => {
+                        if (pattern.includes('*')) {
+                            // Simple glob matching
+                            const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
+                            return regex.test(entry.name);
+                        }
+                        return entry.name === pattern;
+                    });
+
+                    if (!shouldExclude) {
+                        files = files.concat(this.scanDirectory(fullPath, relPath, includePatterns, excludePatterns));
+                    }
+                } else if (entry.isFile()) {
+                    const ext = path.extname(entry.name).toLowerCase();
+
+                    // Check if file should be excluded
+                    const fileExcluded = effectiveExcludes.some(pattern => {
+                        if (pattern.startsWith('*.')) {
+                            return ext === pattern.substring(1);
+                        }
+                        return entry.name === pattern || relPath.includes(pattern);
+                    });
+
+                    if (fileExcluded) continue;
+
+                    // Check if file matches include patterns
+                    let shouldInclude = false;
+                    if (includePatterns && includePatterns.length > 0) {
+                        shouldInclude = includePatterns.some(pattern => {
+                            if (pattern.startsWith('*.')) {
+                                return ext === pattern.substring(1);
+                            }
+                            return entry.name.includes(pattern) || relPath.includes(pattern);
+                        });
+                    } else {
+                        // Use default extensions
+                        shouldInclude = DEFAULT_INCLUDE_EXTS.includes(ext);
+                    }
+
+                    if (shouldInclude) {
+                        try {
+                            const content = fs.readFileSync(fullPath, 'utf8');
+                            // Skip huge files > 500KB
+                            if (content.length < 512 * 1024) {
+                                files.push({
+                                    relativePath: relPath,
+                                    content: content
+                                });
+                            } else {
+                                this.logDebug(`Skipping large file (${Math.round(content.length/1024)}KB): ${relPath}`);
+                            }
+                        } catch (e) {
+                            this.logDebug(`Could not read file: ${relPath}`);
+                        }
+                    }
+                }
+            }
+        } catch (e) {
+            this.logError(`Scan failed for ${dir}:`, e);
+        }
+        return files;
+    }
+
+    /**
+     * Make HTTP request to the Cloudflare Worker (MCP JSON-RPC)
      */
     async makeRequest(message) {
         return new Promise((resolve) => {
             const serverUrl = this.useStreamableHttp ? this.streamableHttpUrl : this.legacyServerUrl;
             const url = new URL(serverUrl);
             const postData = JSON.stringify(message);
-            
+
             const options = {
                 hostname: url.hostname,
                 port: url.port || 443,
@@ -213,12 +467,13 @@ class CFMemoryMCP {
                     'Accept': 'application/json',
                     'Accept-Encoding': 'identity',
                     'User-Agent': this.userAgent,
-                    'X-API-Key': API_KEY,
+                    'Authorization': `Bearer ${API_KEY}`,
+                    'X-API-Key': API_KEY, // Also include for backwards compatibility
                     'Content-Length': Buffer.byteLength(postData)
                 },
                 timeout: TIMEOUT_MS
             };
-            
+
             const req = https.request(options, (res) => {
                 let body = '';
 
@@ -243,7 +498,7 @@ class CFMemoryMCP {
                     }
                 });
             });
-            
+
             req.on('error', (error) => {
                 resolve({
                     jsonrpc: '2.0',
@@ -255,7 +510,7 @@ class CFMemoryMCP {
                     }
                 });
             });
-            
+
             req.on('timeout', () => {
                 req.destroy();
                 resolve({
@@ -268,12 +523,12 @@ class CFMemoryMCP {
                     }
                 });
             });
-            
+
             req.write(postData);
             req.end();
         });
     }
-    
+
     /**
      * Graceful shutdown
      */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cf-memory-mcp",
-	"version": "3.3.0",
+	"version": "3.4.0",
 	"description": "Best-in-class MCP (Model Context Protocol) server for AI memory storage with MIRIX-Inspired Specialized Memory Types (Core, Episodic, Semantic, Procedural, Resource, Knowledge Vault), Progressive Disclosure, AI-Powered Summaries, Context Window Optimization, Summary Memory Feature (TL;DR), Enhanced JSON Processing, Temporal Relationship Tracking, AI-Enhanced Context-Aware + Temporal Intelligence, Smart Tool Recommendation Engine, Memory Intelligence Engine, autonomous optimization, A/B testing, self-improving algorithms, intelligent search, smart auto-features, memory collections, project onboarding workflows, advanced lifecycle management, Project Intelligence for seamless agent handoff, and Unified Project Intelligence Tool with 57% performance improvement",
 	"main": "bin/cf-memory-mcp.js",
 	"bin": {
@@ -131,4 +131,4 @@
 		"typescript": "^5.7.3",
 		"wrangler": "^4.21.2"
 	}
-}
+}