cf-memory-mcp 3.8.2 → 3.8.3

package/README.md

@@ -600,6 +600,77 @@ DEBUG=1 npx cf-memory-mcp
 - `DEBUG=1` - Enable debug logging
 - `MCP_DEBUG=1` - Enable MCP-specific debug logging
 
+## 🧠 NEW: Assistant Memory Features (v4.0.0)
+
+**AI Assistant Memory Management** - Long-term memory for AI assistants with session tracking, context bootstrapping, and entity relationships.
+
+### Assistant Memory Tools
+
+| Tool | Description |
+|------|-------------|
+| `store_memory` | Store facts, preferences, or important context |
+| `retrieve_memories` | Semantic search over memories |
+| `get_context_bootstrap` | Get memories to pre-load on session start |
+| `start_session` | Begin a conversation session |
+| `end_session` | End session, extract key facts |
+| `store_entity` | Store structured entities (people, projects, etc.) |
+
+### Memory Types
+- **fact** - General knowledge about the user
+- **preference** - User preferences and settings
+- **task** - Active or completed tasks
+- **entity** - Structured data about people, projects, companies
+- **session_summary** - Summarized conversation sessions
+
+### Example Usage
+
+```typescript
+// Store a memory
+const result = await store_memory({
+  content: "John is building a RAG pipeline with 4x H100 GPUs",
+  type: "fact",
+  importance: 0.9,
+  confidence: 1.0,
+  source: "user_explicit",
+  tags: ["hardware", "rag", "work"]
+});
+
+// Retrieve relevant memories
+const memories = await retrieve_memories({
+  query: "what hardware is John using?",
+  limit: 5,
+  min_importance: 0.7
+});
+
+// Bootstrap context on session start
+const context = await get_context_bootstrap({
+  max_tokens: 4000,
+  recent_sessions: 3,
+  current_context: "discussing RAG pipeline"
+});
+```
+
+### Database Schema
+
+Run the migration to add assistant memory tables:
+```bash
+wrangler d1 execute MEMORY_DB --file=./migrations/002_assistant_memory.sql
+```
+
+Create the vectorize index:
+```bash
+wrangler vectorize create assistant-memory-index --dimensions=1024 --metric=cosine
+```
+
+Add to `wrangler.toml`:
+```toml
+[[vectorize]]
+binding = "VECTORIZE_ASSISTANT"
+index_name = "assistant-memory-index"
+```
+
+---
+
 ## 📋 Requirements
 
 - **Node.js** 16.0.0 or higher

package/bin/cf-memory-mcp-indexer.js

@@ -0,0 +1,394 @@
+#!/usr/bin/env node
+
+/**
+ * CF-Memory-MCP Local Indexer
+ * 
+ * Fast incremental indexing with local file watching and hash caching.
+ * Only sends changed files to the server, reducing indexing time by 90%+.
+ * 
+ * Usage:
+ *   npx cf-memory-mcp-indexer watch /path/to/project
+ *   npx cf-memory-mcp-indexer index /path/to/project --once
+ * 
+ * Features:
+ * - Local file hash cache (stored in ~/.cache/cf-memory-indexer/)
+ * - Only uploads changed files
+ * - File watching with debouncing
+ * - Batch file uploads (faster than individual file processing)
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const https = require('https');
+const { URL } = require('url');
+const os = require('os');
+const process = require('process');
+
+const API_URL = process.env.CF_MEMORY_API_URL || 'https://cf-memory-mcp-simplified.johnlam90.workers.dev';
+const API_KEY = process.env.CF_MEMORY_API_KEY;
+const CACHE_DIR = path.join(os.homedir(), '.cache', 'cf-memory-indexer');
+
+// File patterns to include/exclude
+const DEFAULT_INCLUDE = [
+  '**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx',
+  '**/*.py', '**/*.go', '**/*.rs', '**/*.java',
+  '**/*.md', '**/*.json'
+];
+const DEFAULT_EXCLUDE = [
+  '**/node_modules/**', '**/.git/**', '**/dist/**',
+  '**/build/**', '**/.next/**', '**/coverage/**',
+  '**/*.min.js', '**/*.d.ts'
+];
+
+class IncrementalIndexer {
+  constructor(projectPath, options = {}) {
+    this.projectPath = path.resolve(projectPath);
+    this.projectName = options.name || path.basename(this.projectPath);
+    this.cacheFile = path.join(CACHE_DIR, `${this.hashString(this.projectPath)}.json`);
+    this.include = options.include || DEFAULT_INCLUDE;
+    this.exclude = options.exclude || DEFAULT_EXCLUDE;
+    this.dryRun = options.dryRun || false;
+    this.batchSize = options.batchSize || 50;
+    
+    // Ensure cache directory exists
+    if (!fs.existsSync(CACHE_DIR)) {
+      fs.mkdirSync(CACHE_DIR, { recursive: true });
+    }
+    
+    this.cache = this.loadCache();
+  }
+
+  hashString(str) {
+    return crypto.createHash('md5').update(str).digest('hex');
+  }
+
+  loadCache() {
+    try {
+      if (fs.existsSync(this.cacheFile)) {
+        return JSON.parse(fs.readFileSync(this.cacheFile, 'utf8'));
+      }
+    } catch (err) {
+      console.error('Failed to load cache:', err.message);
+    }
+    return { files: {}, lastIndexed: null, projectId: null };
+  }
+
+  saveCache() {
+    try {
+      fs.writeFileSync(this.cacheFile, JSON.stringify(this.cache, null, 2));
+    } catch (err) {
+      console.error('Failed to save cache:', err.message);
+    }
+  }
+
+  async scanFiles() {
+    const files = [];
+    const walk = (dir) => {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        const relativePath = path.relative(this.projectPath, fullPath);
+        
+        if (entry.isDirectory()) {
+          // Check if directory should be excluded
+          const shouldExclude = this.exclude.some(pattern => 
+            this.matchGlob(relativePath, pattern.replace('**/', ''))
+          );
+          if (!shouldExclude) {
+            walk(fullPath);
+          }
+        } else {
+          // Check if file should be included
+          const shouldInclude = this.include.some(pattern => 
+            this.matchGlob(relativePath, pattern)
+          );
+          const shouldExclude = this.exclude.some(pattern => 
+            this.matchGlob(relativePath, pattern)
+          );
+          
+          if (shouldInclude && !shouldExclude) {
+            files.push(fullPath);
+          }
+        }
+      }
+    };
+    
+    walk(this.projectPath);
+    return files;
+  }
+
+  matchGlob(filepath, pattern) {
+    // Simple glob matching
+    const regex = new RegExp(
+      '^' + 
+      pattern
+        .replace(/\*\*/g, '<<<GLOBSTAR>>>')
+        .replace(/\*/g, '[^/]*')
+        .replace(/<<<GLOBSTAR>>>/g, '.*')
+        .replace(/\?/g, '.')
+      + '$'
+    );
+    return regex.test(filepath);
+  }
+
+  async getChangedFiles() {
+    const allFiles = await this.scanFiles();
+    const changedFiles = [];
+    const unchangedFiles = [];
+    const newCache = { ...this.cache };
+    
+    for (const filePath of allFiles) {
+      const relativePath = path.relative(this.projectPath, filePath);
+      const stats = fs.statSync(filePath);
+      const mtime = stats.mtime.getTime();
+      const size = stats.size;
+      
+      // Skip files larger than 1MB
+      if (size > 1024 * 1024) {
+        console.log(`⚠️  Skipping large file: ${relativePath}`);
+        continue;
+      }
+      
+      const cached = this.cache.files[relativePath];
+      
+      if (!cached || cached.mtime !== mtime || cached.size !== size) {
+        // File is new or changed
+        const content = fs.readFileSync(filePath, 'utf8');
+        const hash = this.hashString(content);
+        
+        // Double-check with content hash
+        if (!cached || cached.hash !== hash) {
+          changedFiles.push({
+            path: relativePath,
+            fullPath: filePath,
+            content,
+            size,
+            mtime
+          });
+        } else {
+          // Content same but mtime/size changed (git checkout, etc.)
+          unchangedFiles.push(relativePath);
+          newCache.files[relativePath] = { hash, mtime, size };
+        }
+      } else {
+        unchangedFiles.push(relativePath);
+      }
+    }
+    
+    // Detect deleted files
+    const currentFiles = new Set(allFiles.map(f => path.relative(this.projectPath, f)));
+    const deletedFiles = Object.keys(this.cache.files).filter(f => !currentFiles.has(f));
+    
+    return { changedFiles, unchangedFiles, deletedFiles, newCache };
+  }
+
+  async index() {
+    console.log(`🔍 Scanning ${this.projectPath}...`);
+    const { changedFiles, unchangedFiles, deletedFiles, newCache } = await this.getChangedFiles();
+    
+    console.log(`📊 Found ${changedFiles.length} changed, ${unchangedFiles.length} unchanged, ${deletedFiles.length} deleted`);
+    
+    if (changedFiles.length === 0 && deletedFiles.length === 0) {
+      console.log('✅ Everything up to date!');
+      return;
+    }
+    
+    if (this.dryRun) {
+      console.log('🔍 Dry run - would index:');
+      changedFiles.forEach(f => console.log(`  - ${f.path}`));
+      return;
+    }
+    
+    // Upload changed files in batches
+    if (changedFiles.length > 0) {
+      console.log(`📤 Uploading ${changedFiles.length} files...`);
+      await this.uploadFiles(changedFiles);
+    }
+    
+    // Update cache
+    for (const file of changedFiles) {
+      newCache.files[file.path] = {
+        hash: this.hashString(file.content),
+        mtime: file.mtime,
+        size: file.size
+      };
+    }
+    
+    // Remove deleted files from cache
+    for (const file of deletedFiles) {
+      delete newCache.files[file];
+    }
+    
+    newCache.lastIndexed = Date.now();
+    this.cache = newCache;
+    this.saveCache();
+    
+    console.log('✅ Index complete!');
+  }
+
+  async uploadFiles(files) {
+    // Process in batches
+    for (let i = 0; i < files.length; i += this.batchSize) {
+      const batch = files.slice(i, i + this.batchSize);
+      console.log(`  Batch ${Math.floor(i / this.batchSize) + 1}/${Math.ceil(files.length / this.batchSize)} (${batch.length} files)`);
+      
+      await this.uploadBatch(batch);
+    }
+  }
+
+  async uploadBatch(files) {
+    return new Promise((resolve, reject) => {
+      const payload = JSON.stringify({
+        project_path: this.projectPath,
+        project_name: this.projectName,
+        files: files.map(f => ({
+          path: f.path,
+          content: f.content
+        }))
+      });
+      
+      const url = new URL(`${API_URL}/mcp/tools/index_project`);
+      const options = {
+        hostname: url.hostname,
+        path: url.pathname,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${API_KEY}`,
+          'Content-Length': Buffer.byteLength(payload)
+        },
+        timeout: 120000
+      };
+      
+      const req = https.request(options, (res) => {
+        let data = '';
+        res.on('data', chunk => data += chunk);
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            try {
+              const result = JSON.parse(data);
+              if (result.content && result.content[0]) {
+                const content = JSON.parse(result.content[0].text);
+                if (content.project_id) {
+                  this.cache.projectId = content.project_id;
+                }
+              }
+              resolve(result);
+            } catch (err) {
+              reject(new Error(`Invalid response: ${err.message}`));
+            }
+          } else {
+            reject(new Error(`HTTP ${res.statusCode}: ${data}`));
+          }
+        });
+      });
+      
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('Request timeout'));
+      });
+      
+      req.write(payload);
+      req.end();
+    });
+  }
+
+  async watch() {
+    console.log(`👁️  Watching ${this.projectPath} for changes...`);
+    console.log('Press Ctrl+C to stop\n');
+    
+    // Initial index
+    await this.index();
+    
+    // Watch for changes
+    const chokidar = await this.loadChokidar();
+    
+    const watcher = chokidar.watch(this.projectPath, {
+      ignored: this.exclude,
+      persistent: true,
+      ignoreInitial: true
+    });
+    
+    let debounceTimer = null;
+    
+    const handleChange = async () => {
+      if (debounceTimer) clearTimeout(debounceTimer);
+      debounceTimer = setTimeout(async () => {
+        console.log('\n🔄 Changes detected, re-indexing...');
+        await this.index();
+        console.log('');
+      }, 1000);
+    };
+    
+    watcher
+      .on('add', path => handleChange())
+      .on('change', path => handleChange())
+      .on('unlink', path => handleChange());
+    
+    // Keep process alive
+    process.on('SIGINT', () => {
+      console.log('\n👋 Stopping watcher...');
+      watcher.close();
+      process.exit(0);
+    });
+  }
+
+  async loadChokidar() {
+    try {
+      return require('chokidar');
+    } catch (err) {
+      console.log('Installing chokidar for file watching...');
+      const { execSync } = require('child_process');
+      execSync('npm install chokidar --save-dev', { cwd: __dirname, stdio: 'inherit' });
+      return require('chokidar');
+    }
+  }
+}
+
+// CLI
+async function main() {
+  if (!API_KEY) {
+    console.error('❌ CF_MEMORY_API_KEY environment variable required');
+    process.exit(1);
+  }
+  
+  const args = process.argv.slice(2);
+  const command = args[0];
+  const projectPath = args[1] || '.';
+  
+  const options = {
+    dryRun: args.includes('--dry-run'),
+    once: args.includes('--once')
+  };
+  
+  const indexer = new IncrementalIndexer(projectPath, options);
+  
+  if (command === 'watch') {
+    await indexer.watch();
+  } else if (command === 'index') {
+    await indexer.index();
+  } else {
+    console.log(`
+CF-Memory-MCP Incremental Indexer
+
+Usage:
+  cf-memory-mcp-indexer index <path>    Index project once
+  cf-memory-mcp-indexer watch <path>    Watch and auto-index on changes
+
+Options:
+  --dry-run    Show what would be indexed without uploading
+  
+Environment:
+  CF_MEMORY_API_KEY    Required. Your API key
+  CF_MEMORY_API_URL    Optional. Custom API endpoint
+    `);
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('Error:', err.message);
+  process.exit(1);
+});

package/package.json

@@ -1,10 +1,12 @@
 {
 	"name": "cf-memory-mcp",
-	"version": "3.8.2",
+	"version": "3.8.3",
 	"description": "Best-in-class MCP server with CONTEXTUAL CHUNKING (Anthropic-style, 35-67% better retrieval), Optimized LLM stack (Llama-3.1-8B), BGE-M3 embeddings, Query Expansion Caching, Hybrid Embedding Strategy, and Unified Project Intelligence",
 	"main": "bin/cf-memory-mcp.js",
 	"bin": {
-		"cf-memory-mcp": "bin/cf-memory-mcp.js"
+		"cf-memory-mcp": "bin/cf-memory-mcp.js",
+		"cf-memory-index": "bin/cf-memory-mcp-indexer.js",
+		"cf-memory-watch": "bin/cf-memory-mcp-indexer.js"
 	},
 	"author": {
 		"name": "John Lam",
@@ -130,6 +132,6 @@
 		"jest-environment-node": "^30.0.4",
 		"ts-jest": "^29.4.0",
 		"typescript": "^5.7.3",
-		"wrangler": "^4.21.2"
+		"wrangler": "^4.67.1"
 	}
 }