cf-memory-mcp 3.8.4 → 3.8.6

package/bin/cf-memory-mcp-indexer.js

@@ -25,20 +25,51 @@ const { URL } = require('url');
 const os = require('os');
 const process = require('process');
 
-const API_URL = process.env.CF_MEMORY_API_URL || 'https://cf-memory-mcp-simplified.johnlam90.workers.dev';
+// Resolve API URL, overriding the dead workers.dev URL if it's still in the
+// shell environment from before the memcp.ai migration.
+function _resolveApiUrl() {
+    const raw = process.env.CF_MEMORY_API_URL || process.env.CF_MEMORY_BASE_URL;
+    if (!raw || raw.includes('workers.dev')) {
+        return 'https://memcp.ai';
+    }
+    return raw;
+}
+const API_URL = _resolveApiUrl();
 const API_KEY = process.env.CF_MEMORY_API_KEY;
 const CACHE_DIR = path.join(os.homedir(), '.cache', 'cf-memory-indexer');
 
 // File patterns to include/exclude
+// Mirrors src-simplified/utils/index.ts DEFAULT_INCLUDE_EXTENSIONS (110+ languages)
 const DEFAULT_INCLUDE = [
-  '**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx',
-  '**/*.py', '**/*.go', '**/*.rs', '**/*.java',
-  '**/*.md', '**/*.json'
+  // Mainstream
+  '**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.mjs', '**/*.cjs',
+  '**/*.py', '**/*.rb', '**/*.go', '**/*.rs', '**/*.java', '**/*.kt', '**/*.scala',
+  '**/*.cs', '**/*.cpp', '**/*.c', '**/*.h', '**/*.hpp', '**/*.swift', '**/*.php',
+  // Functional & systems
+  '**/*.lua', '**/*.ex', '**/*.exs', '**/*.hs', '**/*.lhs',
+  '**/*.clj', '**/*.cljs', '**/*.cljc', '**/*.jl', '**/*.elm', '**/*.purs',
+  '**/*.dart', '**/*.zig', '**/*.nim', '**/*.cr', '**/*.erl', '**/*.hrl',
+  '**/*.ml', '**/*.fs', '**/*.fsi',
+  // Web & data
+  '**/*.html', '**/*.css', '**/*.scss', '**/*.vue', '**/*.svelte', '**/*.astro',
+  '**/*.json', '**/*.yaml', '**/*.yml', '**/*.toml',
+  '**/*.md', '**/*.mdx',
+  // Infra & build
+  '**/*.tf', '**/*.hcl', '**/*.sql', '**/*.sh', '**/*.bash',
+  '**/*.proto', '**/*.graphql', '**/*.gql',
+  // Shaders & GPU
+  '**/*.glsl', '**/*.hlsl', '**/*.wgsl', '**/*.metal', '**/*.cu', '**/*.cuh',
 ];
+
 const DEFAULT_EXCLUDE = [
   '**/node_modules/**', '**/.git/**', '**/dist/**',
-  '**/build/**', '**/.next/**', '**/coverage/**',
-  '**/*.min.js', '**/*.d.ts'
+  '**/build/**', '**/out/**', '**/.next/**', '**/.nuxt/**',
+  '**/coverage/**', '**/__pycache__/**', '**/venv/**', '**/.venv/**',
+  '**/.wrangler/**', '**/.turbo/**', '**/.cache/**',
+  '**/.history/**', '**/.vscode/**', '**/.idea/**', '**/.vs/**',
+  '**/.augment/**', '**/.kiro/**', '**/.intent/**', '**/.husky/**', '**/.claude/**',
+  '**/*.min.js', '**/*.min.css', '**/*.map', '**/*.d.ts',
+  '**/package-lock.json', '**/yarn.lock', '**/pnpm-lock.yaml',
 ];
 
 class IncrementalIndexer {
@@ -205,6 +236,16 @@ class IncrementalIndexer {
       console.log(`📤 Uploading ${changedFiles.length} files...`);
       await this.uploadFiles(changedFiles);
     }
+
+    if (this.cache.projectId && deletedFiles.length > 0) {
+      console.log(`🧹 Cleaning up ${deletedFiles.length} deleted files...`);
+      await this.cleanupDeletedFiles(deletedFiles);
+    }
+
+    if (this.cache.projectId && (changedFiles.length > 0 || deletedFiles.length > 0)) {
+      console.log('📌 Finalizing project stats...');
+      await this.finalizeProject();
+    }
     
     // Update cache
     for (const file of changedFiles) {
@@ -238,59 +279,101 @@ class IncrementalIndexer {
   }
 
   async uploadBatch(files) {
-    return new Promise((resolve, reject) => {
-      const payload = JSON.stringify({
+    if (!this.cache.projectId) {
+      const initResult = await this.callMcpTool('index_project', {
         project_path: this.projectPath,
-        project_name: this.projectName,
-        files: files.map(f => ({
-          path: f.path,
-          content: f.content
-        }))
+        project_name: this.projectName
       });
-      
-      const url = new URL(`${API_URL}/mcp/tools/index_project`);
+      const initContent = this.parseToolTextResult(initResult);
+
+      if (!initContent.project_id) {
+        throw new Error('Project initialization did not return project_id');
+      }
+
+      this.cache.projectId = initContent.project_id;
+      this.saveCache();
+    }
+
+    return this.postJson(`/api/projects/${encodeURIComponent(this.cache.projectId)}/files/batch`, {
+      files: files.map(f => ({
+        file_path: f.path,
+        content: f.content
+      }))
+    });
+  }
+
+  async callMcpTool(name, args) {
+    return this.postJson('/mcp', {
+      jsonrpc: '2.0',
+      id: `idx-${Date.now()}`,
+      method: 'tools/call',
+      params: {
+        name,
+        arguments: args
+      }
+    });
+  }
+
+  async cleanupDeletedFiles(filePaths) {
+    return this.postJson(`/api/projects/${encodeURIComponent(this.cache.projectId)}/files/cleanup`, {
+      file_paths: filePaths
+    });
+  }
+
+  async finalizeProject() {
+    return this.postJson(`/api/projects/${encodeURIComponent(this.cache.projectId)}/finalize`, {});
+  }
+
+  parseToolTextResult(result) {
+    const text = result?.result?.content?.[0]?.text;
+    if (!text) {
+      throw new Error('Tool response did not contain text content');
+    }
+    return JSON.parse(text);
+  }
+
+  async postJson(route, payload) {
+    return new Promise((resolve, reject) => {
+      const body = JSON.stringify(payload);
+      const url = new URL(`${API_URL}${route}`);
       const options = {
         hostname: url.hostname,
-        path: url.pathname,
+        port: url.port || 443,
+        path: url.pathname + url.search,
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
           'Authorization': `Bearer ${API_KEY}`,
-          'Content-Length': Buffer.byteLength(payload)
+          'X-API-Key': API_KEY,
+          'Content-Length': Buffer.byteLength(body)
         },
         timeout: 120000
       };
-      
+
       const req = https.request(options, (res) => {
         let data = '';
         res.on('data', chunk => data += chunk);
         res.on('end', () => {
-          if (res.statusCode >= 200 && res.statusCode < 300) {
-            try {
-              const result = JSON.parse(data);
-              if (result.content && result.content[0]) {
-                const content = JSON.parse(result.content[0].text);
-                if (content.project_id) {
-                  this.cache.projectId = content.project_id;
-                }
-              }
-              resolve(result);
-            } catch (err) {
-              reject(new Error(`Invalid response: ${err.message}`));
+          try {
+            const parsed = JSON.parse(data || '{}');
+            if (res.statusCode >= 200 && res.statusCode < 300) {
+              resolve(parsed);
+              return;
             }
-          } else {
             reject(new Error(`HTTP ${res.statusCode}: ${data}`));
+          } catch (err) {
+            reject(new Error(`Invalid response: ${err.message}`));
           }
         });
       });
-      
+
       req.on('error', reject);
       req.on('timeout', () => {
         req.destroy();
         reject(new Error('Request timeout'));
       });
-      
-      req.write(payload);
+
+      req.write(body);
       req.end();
     });
   }
@@ -302,49 +385,41 @@ class IncrementalIndexer {
     // Initial index
     await this.index();
     
-    // Watch for changes
-    const chokidar = await this.loadChokidar();
-    
-    const watcher = chokidar.watch(this.projectPath, {
-      ignored: this.exclude,
-      persistent: true,
-      ignoreInitial: true
-    });
-    
-    let debounceTimer = null;
-    
-    const handleChange = async () => {
-      if (debounceTimer) clearTimeout(debounceTimer);
-      debounceTimer = setTimeout(async () => {
-        console.log('\n🔄 Changes detected, re-indexing...');
-        await this.index();
-        console.log('');
-      }, 1000);
+    // Use Node's built-in fs.watch (no external deps needed)
+    const fs = require('fs');
+    const watchDir = (dir) => {
+      let watcher;
+      try {
+        watcher = fs.watch(dir, { recursive: true }, (eventType, filename) => {
+          if (filename) {
+            this.handleFileChange(dir, filename);
+          }
+        });
+      } catch (err) {
+        // Ignore errors for now
+      }
+      return watcher;
     };
     
-    watcher
-      .on('add', path => handleChange())
-      .on('change', path => handleChange())
-      .on('unlink', path => handleChange());
+    const watcher = watchDir(this.projectPath);
     
     // Keep process alive
     process.on('SIGINT', () => {
       console.log('\n👋 Stopping watcher...');
-      watcher.close();
+      if (watcher) watcher.close();
       process.exit(0);
     });
   }
 
-  async loadChokidar() {
-    try {
-      return require('chokidar');
-    } catch (err) {
-      console.log('Installing chokidar for file watching...');
-      const { execSync } = require('child_process');
-      execSync('npm install chokidar --save-dev', { cwd: __dirname, stdio: 'inherit' });
-      return require('chokidar');
-    }
+  handleFileChange(baseDir, filename) {
+    // Debounce rapid changes
+    if (this._debounceTimer) clearTimeout(this._debounceTimer);
+    this._debounceTimer = setTimeout(async () => {
+      console.log(`\n🔄 File changed: ${filename}`);
+      await this.index();
+    }, 1000);
   }
+
 }
 
 // CLI
@@ -358,15 +433,20 @@ async function main() {
   
   // Detect command from how the script was called (cf-memory-index vs cf-memory-watch)
   const scriptName = path.basename(process.argv[1]);
-  let command = args[0];
-  let projectPath = args[1] || '.';
+  let command = 'index';
+  let projectPath = '.';
   
-  if (scriptName.includes('watch')) {
+  if (args[0] === 'watch' || args[0] === 'index') {
+    command = args[0];
+    projectPath = args[1] || '.';
+  } else if (scriptName.includes('watch')) {
     command = 'watch';
     projectPath = args[0] || '.';
   } else if (scriptName.includes('index')) {
     command = 'index';
     projectPath = args[0] || '.';
+  } else {
+    projectPath = args[0] || '.';
   }
   
   const options = {