cf-memory-mcp 3.9.8 → 3.9.11

package/README.md

@@ -8,9 +8,9 @@ Cloudflare-hosted MCP server for semantic code indexing, retrieval, and assistan
 
 - **110+ language support** - TypeScript, Python, Go, Rust, Java, C/C++, Ruby, PHP, Swift, Kotlin, Scala, and 100+ more including shaders (GLSL, HLSL, WGSL, Metal, CUDA)
 - **Hybrid retrieval** - 3-lane RRF (semantic + keyword + name matching) with cross-encoder reranking
-- **100% benchmark accuracy** - 32 queries across behavioral, conceptual, identifier, and negation categories
+- **100% top-3 benchmark accuracy** - 37 queries across behavioral, conceptual, identifier, and negation categories
 - **Code graph navigation** - Track calls, imports, inheritance across 9 language families (TS/JS, Python, Go, Rust, Java/Kotlin/Scala, C/C++/C#, Ruby, PHP, Swift)
-- **21 MCP tools** - Indexing, retrieval, file exploration, code graph, freshness management, and persistent memory
+- **22 MCP tools** - Indexing, retrieval, file exploration, code graph, freshness management, indexing status, and persistent memory
 - **Self-healing freshness** - Results auto-flag stale chunks with a copy-pasteable `refresh_files` call; `refresh_stale` rebuilds the changed files in seconds; cache invalidates on write
 - **Self-debugging errors** - `retrieve_context` includes `empty_hint` on 0-result queries; `get_file_content` / `get_file_outline` return `{error, hint}` pointing to the next call instead of bare null; calling bridge-only tools server-side returns `bridge_required` install instructions
 - **Smart defaults** - Bridge auto-detects project_id from cwd, returns chunks pre-enriched with file imports + citation + source classification
@@ -22,7 +22,7 @@ The active runtime is at [src-simplified/index.ts](src-simplified/index.ts). It
 
 - Direct remote MCP on `/mcp`
 - Legacy compatibility on `/mcp/message`
-- Code indexing with `index_project` and `index_github`
+- Code indexing with `index_project` and `index_github`; local bridge indexing returns immediately with a `bridge_job_id`, then `get_indexing_status` reports progress
 - Retrieval with `retrieve_context` (3-lane hybrid + cross-encoder rerank, returns `file_imports` + `source_kind` + `citation`)
 - Code relationship graph with `get_related_code` (calls, imports, extends, implements)
 - Project exploration with `list_files`, `list_projects`, `get_stats`, `get_file_outline`, `get_file_content` (bridge reads local file byte-exact when resolvable; falls back to chunk-reconstruction with `reconstruction_warning` + `missing_line_ranges`)

package/bin/cf-memory-mcp.js

@@ -102,11 +102,24 @@ const TOOLS_LIST = [
             properties: {
                 project_path: { type: 'string', description: 'Absolute path to the project root directory' },
                 project_name: { type: 'string', description: 'Display name for the project (defaults to directory basename)' },
-                force_reindex: { type: 'boolean', description: 'If true, wipes existing chunks and rebuilds from scratch. Use only when needed; incremental is much faster.' }
+                force_reindex: { type: 'boolean', description: 'If true, wipes existing chunks and rebuilds from scratch. Use only when needed; incremental is much faster.' },
+                allow_system_path: { type: 'boolean', description: 'Allow indexing a path outside the current workspace root. Defaults false for safety.' },
+                wait_for_completion: { type: 'boolean', description: 'If true, block until indexing finishes. Default false: return immediately with bridge_job_id and poll get_indexing_status.' }
             },
             required: ['project_path']
         }
     },
+    {
+        name: 'get_indexing_status',
+        description: 'Check status for an index_project background job. Use bridge_job_id returned by index_project, or project_id as a fallback. Do not poll list_projects for indexing progress.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                bridge_job_id: { type: 'string', description: 'Bridge-local job id returned by index_project' },
+                project_id: { type: 'string', description: 'Project ID or name to inspect when bridge_job_id is unavailable' }
+            }
+        }
+    },
     {
         name: 'index_github',
         description: 'Index a public GitHub repository server-side without cloning locally. Use when the user wants to search a remote codebase.',
@@ -369,14 +382,7 @@ class CFMemoryMCP {
         this.userAgent = `cf-memory-mcp/${PACKAGE_VERSION} (${os.platform()} ${os.arch()}; Node.js ${process.version})`;
         this.useStreamableHttp = true; // Try Streamable HTTP first
         this.autoWatcher = null; // Background file watcher
-
-        // Handle process termination gracefully
-        process.on('SIGINT', () => this.shutdown('SIGINT'));
-        process.on('SIGTERM', () => this.shutdown('SIGTERM'));
-        process.on('uncaughtException', (error) => {
-            this.logError('Uncaught exception:', error);
-            this.shutdown('ERROR');
-        });
+        this.indexJobs = new Map(); // bridge_job_id -> local background indexing status
 
         // Set up stdio encoding
         process.stdin.setEncoding('utf8');
@@ -414,6 +420,7 @@ class CFMemoryMCP {
     async start() {
         try {
             this.logDebug('Starting MCP message processing...');
+            this.installProcessHandlers();
 
             // Pre-warm the HTTPS connection in the background so the first
             // real tool call doesn't pay the TLS handshake cost.
@@ -436,6 +443,18 @@ class CFMemoryMCP {
         }
     }
 
+    installProcessHandlers() {
+        if (CFMemoryMCP._processHandlersInstalled) return;
+        CFMemoryMCP._processHandlersInstalled = true;
+
+        process.on('SIGINT', () => this.shutdown('SIGINT'));
+        process.on('SIGTERM', () => this.shutdown('SIGTERM'));
+        process.on('uncaughtException', (error) => {
+            this.logError('Uncaught exception:', error);
+            this.shutdown('ERROR');
+        });
+    }
+
     /**
      * Resolve the current cwd to a project_id in the background and
      * cache the result, so the first retrieve_context query that
@@ -725,6 +744,14 @@ class CFMemoryMCP {
                 return;
             }
 
+            // Intercept get_indexing_status: bridge-local background index
+            // jobs are not visible to the remote worker until batches land,
+            // so status must be served by this bridge process when possible.
+            if (message.method === 'tools/call' && message.params && message.params.name === 'get_indexing_status') {
+                await this.handleGetIndexingStatus(message);
+                return;
+            }
+
             // Intercept index_project tool call to perform local scanning
             if (message.method === 'tools/call' && message.params && message.params.name === 'index_project') {
                 await this.handleIndexProject(message);
@@ -1169,7 +1196,7 @@ class CFMemoryMCP {
 
     async handleIndexProject(message) {
         const args = message.params?.arguments || {};
-        const { project_path, project_name, include_patterns, exclude_patterns, force_reindex } = args;
+        const { project_path, project_name, include_patterns, exclude_patterns, force_reindex, allow_system_path, wait_for_completion } = args;
 
         // Boundary validation: bad inputs were producing unhelpful Node errors
         // ("path must be string"). Return a clean MCP error with the hint
@@ -1188,6 +1215,32 @@ class CFMemoryMCP {
 
         const resolvedPath = path.resolve(project_path);
         const name = project_name || path.basename(resolvedPath);
+        const workspaceRoot = (() => {
+            const root = process.env.CF_MEMORY_WATCH_PATH || process.cwd();
+            try { return fs.realpathSync(path.resolve(root)); }
+            catch (_) { return path.resolve(root); }
+        })();
+        const resolvedRealPath = (() => {
+            try { return fs.realpathSync(resolvedPath); }
+            catch (_) { return resolvedPath; }
+        })();
+        const relativeToWorkspace = path.relative(workspaceRoot, resolvedRealPath);
+        const insideWorkspace = relativeToWorkspace && !relativeToWorkspace.startsWith('..') && !path.isAbsolute(relativeToWorkspace);
+        const sameAsWorkspace = relativeToWorkspace === '';
+
+        if (!allow_system_path && !insideWorkspace && !sameAsWorkspace) {
+            process.stdout.write(JSON.stringify({
+                jsonrpc: '2.0',
+                id: message.id,
+                result: { content: [{ type: 'text', text: JSON.stringify({
+                    error: 'project_path_outside_workspace',
+                    project_path: resolvedPath,
+                    workspace_root: workspaceRoot,
+                    hint: 'index_project is confined to the current workspace root by default. Run the bridge from that project directory, set CF_MEMORY_WATCH_PATH to that root, or pass allow_system_path:true if you intentionally want to index another readable local directory.',
+                }, null, 2) }] },
+            }) + '\n');
+            return;
+        }
 
         this.logDebug(`Intercepted index_project for: ${resolvedPath} (${name})`);
 
@@ -1244,8 +1297,6 @@ class CFMemoryMCP {
 
             // 2. Create project via MCP (no inline files), then upload files via batch endpoint.
             // This avoids repeating full index_project for every file batch.
-            let totalIndexed = 0;
-            let totalChunks = 0;
             let projectId = null;
 
             const init = await this.makeRequest({
@@ -1280,84 +1331,54 @@ class CFMemoryMCP {
             // Adaptive batching: limit by file count and payload bytes to reduce timeouts.
             const batches = this.createAdaptiveBatches(files);
 
-            let totalSkipped = 0;
-            let totalUnchanged = 0;
-            const batchErrors = [];
-            const failedBatches = [];
-
-            // Process batches in parallel (concurrency = 3). Each batch is
-            // independent on the server side, so overlapping them gives a
-            // ~3x speedup for large projects. Higher concurrency risks
-            // overwhelming the Cloudflare Worker / hitting per-account
-            // request limits, so we cap conservatively.
-            const CONCURRENCY = Math.min(3, batches.length);
-            const aggregateBatchResult = (uploadResult, b, batch) => {
-                if (!uploadResult) {
-                    failedBatches.push({ batch: b + 1, files: batch.length, reason: 'no response (timeout/network)' });
-                    this.logError(`Batch ${b + 1}/${batches.length} returned no response (timeout or network error). ${batch.length} files unaccounted for.`);
-                    return;
-                }
-                if (typeof uploadResult.files_indexed === 'number') {
-                    totalIndexed += uploadResult.files_indexed;
-                }
-                if (typeof uploadResult.chunks_created === 'number') {
-                    totalChunks += uploadResult.chunks_created;
-                }
-                if (typeof uploadResult.files_skipped === 'number') {
-                    totalSkipped += uploadResult.files_skipped;
-                }
-                if (typeof uploadResult.files_unchanged === 'number') {
-                    totalUnchanged += uploadResult.files_unchanged;
-                }
-                if (Array.isArray(uploadResult.errors) && uploadResult.errors.length > 0) {
-                    for (const err of uploadResult.errors) {
-                        batchErrors.push(err);
-                    }
-                }
-            };
+            const bridgeJobId = `bridge_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
+            const job = this.createLocalIndexJob({
+                bridgeJobId,
+                projectId,
+                projectName: name,
+                projectPath: resolvedPath,
+                filesFound: files.length,
+                batchesTotal: batches.length,
+            });
 
-            for (let i = 0; i < batches.length; i += CONCURRENCY) {
-                const window = [];
-                for (let j = i; j < Math.min(i + CONCURRENCY, batches.length); j++) {
-                    const batch = batches[j];
-                    const approxBytes = batch.reduce((sum, f) => sum + Buffer.byteLength(f.content || '', 'utf8'), 0);
-                    this.logDebug(`Uploading batch ${j + 1}/${batches.length} (${batch.length} files, ~${Math.round(approxBytes / 1024)}KB)`);
-                    window.push(
-                        this.uploadFileBatch(projectId, batch).then(res => aggregateBatchResult(res, j, batch))
-                    );
-                }
-                await Promise.all(window);
-            }
+            const runPromise = this.runLocalIndexJob(bridgeJobId, {
+                projectId,
+                files,
+                batches,
+                progressStream,
+            });
+            // Ownership transferred to runLocalIndexJob; do not close it in
+            // this request's finally block when the default async mode returns.
+            progressStream = null;
 
-            // 3. Cleanup stale files (accuracy): remove server-side files not present locally.
-            // Best-effort: failure shouldn't fail indexing.
-            if (projectId) {
-                try {
-                    await this.cleanupStaleFiles(projectId, files.map(f => f.relativePath));
-                } catch (e) {
-                    this.logDebug(`Cleanup stale files failed: ${e && e.message ? e.message : 'unknown error'}`);
-                }
+            if (wait_for_completion === true) {
+                await runPromise;
+                const responsePayload = this.serializeIndexJob(job);
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: {
+                        content: [{
+                            type: 'text',
+                            text: JSON.stringify(responsePayload, null, 2)
+                        }]
+                    }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
             }
 
-            // 4. Return aggregated success (with skipped/error visibility)
-            const status = failedBatches.length > 0 ? 'partial' : 'complete';
             const responsePayload = {
                 project_id: projectId,
                 project_name: name,
+                bridge_job_id: bridgeJobId,
                 files_found: files.length,
-                files_indexed: totalIndexed,
-                files_unchanged: totalUnchanged,
-                files_skipped: totalSkipped,
-                chunks_created: totalChunks,
-                status
+                batches_queued: batches.length,
+                status: 'indexing',
+                async: true,
+                message: 'Indexing started in the bridge background. Do not poll list_projects; call get_indexing_status with bridge_job_id.',
+                progress_hint: `Call get_indexing_status with bridge_job_id="${bridgeJobId}" until status is "complete", "partial", or "failed". Retrieval before completion may be partial.`
             };
-            if (batchErrors.length > 0) {
-                responsePayload.errors = batchErrors.slice(0, 50);
-                responsePayload.errors_truncated = batchErrors.length > 50;
-            }
-            if (failedBatches.length > 0) {
-                responsePayload.failed_batches = failedBatches;
-            }
             const response = {
                 jsonrpc: '2.0',
                 id: message.id,
@@ -1389,6 +1410,177 @@ class CFMemoryMCP {
         }
     }
 
+    createLocalIndexJob({ bridgeJobId, projectId, projectName, projectPath, filesFound, batchesTotal }) {
+        const now = new Date().toISOString();
+        const job = {
+            bridge_job_id: bridgeJobId,
+            project_id: projectId,
+            project_name: projectName,
+            project_path: projectPath,
+            status: 'indexing',
+            async: true,
+            files_found: filesFound,
+            files_indexed: 0,
+            files_unchanged: 0,
+            files_skipped: 0,
+            chunks_created: 0,
+            batches_total: batchesTotal,
+            batches_completed: 0,
+            failed_batches: [],
+            errors: [],
+            started_at: now,
+            updated_at: now,
+            completed_at: null,
+            finalized: false,
+        };
+        this.indexJobs.set(bridgeJobId, job);
+        return job;
+    }
+
+    serializeIndexJob(job) {
+        const errors = Array.isArray(job.errors) ? job.errors : [];
+        const failedBatches = Array.isArray(job.failed_batches) ? job.failed_batches : [];
+        return {
+            bridge_job_id: job.bridge_job_id,
+            project_id: job.project_id,
+            project_name: job.project_name,
+            project_path: job.project_path,
+            status: job.status,
+            async: true,
+            progress: {
+                batches_total: job.batches_total,
+                batches_completed: job.batches_completed,
+                percentage: job.batches_total > 0
+                    ? Math.round((job.batches_completed / job.batches_total) * 100)
+                    : 100,
+            },
+            files_found: job.files_found,
+            files_indexed: job.files_indexed,
+            files_unchanged: job.files_unchanged,
+            files_skipped: job.files_skipped,
+            chunks_created: job.chunks_created,
+            finalized: job.finalized,
+            started_at: job.started_at,
+            updated_at: job.updated_at,
+            completed_at: job.completed_at,
+            errors: errors.length > 0 ? errors.slice(0, 50) : undefined,
+            errors_truncated: errors.length > 50 ? true : undefined,
+            failed_batches: failedBatches.length > 0 ? failedBatches : undefined,
+            hint: job.status === 'indexing'
+                ? `Still indexing. Poll get_indexing_status with bridge_job_id="${job.bridge_job_id}" later; do not poll list_projects.`
+                : 'Indexing finished. retrieve_context can now use the completed index.',
+        };
+    }
+
+    async runLocalIndexJob(bridgeJobId, { projectId, files, batches, progressStream }) {
+        const job = this.indexJobs.get(bridgeJobId);
+        if (!job) return null;
+
+        const touch = () => { job.updated_at = new Date().toISOString(); };
+        const aggregateBatchResult = (uploadResult, b, batch) => {
+            if (!uploadResult) {
+                job.failed_batches.push({ batch: b + 1, files: batch.length, reason: 'no response (timeout/network)' });
+                this.logError(`Batch ${b + 1}/${batches.length} returned no response (timeout or network error). ${batch.length} files unaccounted for.`);
+            } else {
+                if (typeof uploadResult.files_indexed === 'number') {
+                    job.files_indexed += uploadResult.files_indexed;
+                }
+                if (typeof uploadResult.chunks_created === 'number') {
+                    job.chunks_created += uploadResult.chunks_created;
+                }
+                if (typeof uploadResult.files_skipped === 'number') {
+                    job.files_skipped += uploadResult.files_skipped;
+                }
+                if (typeof uploadResult.files_unchanged === 'number') {
+                    job.files_unchanged += uploadResult.files_unchanged;
+                }
+                if (Array.isArray(uploadResult.errors) && uploadResult.errors.length > 0) {
+                    for (const err of uploadResult.errors) {
+                        job.errors.push(err);
+                    }
+                }
+            }
+            job.batches_completed += 1;
+            touch();
+        };
+
+        try {
+            const CONCURRENCY = Math.min(3, batches.length);
+            for (let i = 0; i < batches.length; i += CONCURRENCY) {
+                const window = [];
+                for (let j = i; j < Math.min(i + CONCURRENCY, batches.length); j++) {
+                    const batch = batches[j];
+                    const approxBytes = batch.reduce((sum, f) => sum + Buffer.byteLength(f.content || '', 'utf8'), 0);
+                    this.logDebug(`Uploading batch ${j + 1}/${batches.length} (${batch.length} files, ~${Math.round(approxBytes / 1024)}KB)`);
+                    window.push(
+                        this.uploadFileBatch(projectId, batch).then(res => aggregateBatchResult(res, j, batch))
+                    );
+                }
+                await Promise.all(window);
+            }
+
+            try {
+                await this.cleanupStaleFiles(projectId, files.map(f => f.relativePath));
+            } catch (e) {
+                this.logDebug(`Cleanup stale files failed: ${e && e.message ? e.message : 'unknown error'}`);
+            }
+
+            const finalizeResult = await this.finalizeProject(projectId);
+            if (finalizeResult && finalizeResult.success) {
+                job.finalized = true;
+            } else {
+                job.errors.push(`finalize failed: ${finalizeResult && finalizeResult.error ? finalizeResult.error : 'no response'}`);
+            }
+
+            job.status = job.failed_batches.length > 0 || job.errors.length > 0 ? 'partial' : 'complete';
+            job.completed_at = new Date().toISOString();
+            touch();
+        } catch (error) {
+            job.status = 'failed';
+            job.errors.push(error && error.message ? error.message : String(error));
+            job.completed_at = new Date().toISOString();
+            touch();
+            this.logError('Background index job failed:', error);
+        } finally {
+            if (progressStream && typeof progressStream.stop === 'function') {
+                try { progressStream.stop(); } catch (_) {}
+            }
+        }
+
+        return job;
+    }
+
+    async handleGetIndexingStatus(message) {
+        const args = message.params?.arguments || {};
+        const { bridge_job_id, project_id } = args;
+        let payload = null;
+
+        if (bridge_job_id && this.indexJobs.has(bridge_job_id)) {
+            payload = this.serializeIndexJob(this.indexJobs.get(bridge_job_id));
+        } else if (project_id) {
+            const localJob = Array.from(this.indexJobs.values())
+                .reverse()
+                .find((job) => job.project_id === project_id || job.project_name === project_id);
+
+            if (localJob) {
+                payload = this.serializeIndexJob(localJob);
+            } else {
+                payload = await this.getRemoteProjectStatus(project_id);
+            }
+        } else {
+            payload = {
+                error: 'bridge_job_id or project_id is required',
+                hint: 'Pass bridge_job_id returned by index_project. If unavailable, pass project_id.',
+            };
+        }
+
+        process.stdout.write(JSON.stringify({
+            jsonrpc: '2.0',
+            id: message.id,
+            result: { content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }] },
+        }) + '\n');
+    }
+
     /**
      * Recursive directory scan with filtering
      */
@@ -1681,6 +1873,98 @@ class CFMemoryMCP {
         });
     }
 
+    async finalizeProject(projectId) {
+        const url = new URL(this.legacyServerUrl);
+        const finalizePath = `/api/projects/${encodeURIComponent(projectId)}/finalize`;
+        const postData = JSON.stringify({});
+
+        return new Promise((resolve) => {
+            const headers = {
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'Accept-Encoding': 'identity',
+                'User-Agent': this.userAgent,
+                'Authorization': `Bearer ${API_KEY}`,
+                'X-API-Key': API_KEY,
+                'Content-Length': Buffer.byteLength(postData)
+            };
+
+            const options = {
+                hostname: url.hostname,
+                port: url.port || 443,
+                path: finalizePath,
+                method: 'POST',
+                headers,
+                timeout: BATCH_TIMEOUT_MS,
+                agent: httpsAgent
+            };
+
+            const req = https.request(options, (res) => {
+                let body = '';
+                res.on('data', (chunk) => { body += chunk; });
+                res.on('end', () => {
+                    try {
+                        const parsed = JSON.parse(body);
+                        if (res.statusCode && res.statusCode >= 400) {
+                            resolve({ error: parsed.error || `HTTP ${res.statusCode}` });
+                        } else {
+                            resolve(parsed);
+                        }
+                    } catch (_) {
+                        resolve({ error: `Invalid JSON response (HTTP ${res.statusCode})` });
+                    }
+                });
+            });
+
+            req.on('error', (err) => resolve({ error: err.message }));
+            req.on('timeout', () => { try { req.destroy(); } catch (_) {} resolve({ error: 'timeout' }); });
+            req.write(postData);
+            req.end();
+        });
+    }
+
+    async getRemoteProjectStatus(projectIdOrName) {
+        const response = await this.makeRequest({
+            jsonrpc: '2.0',
+            id: `status-${Date.now()}`,
+            method: 'tools/call',
+            params: { name: 'list_projects', arguments: {} },
+        });
+
+        try {
+            const parsed = JSON.parse(response.result.content[0].text);
+            const projects = parsed.projects || parsed.results || [];
+            const project = projects.find((p) => p.id === projectIdOrName || p.project_id === projectIdOrName || p.name === projectIdOrName);
+            if (!project) {
+                return {
+                    error: 'project_not_found',
+                    project_id: projectIdOrName,
+                    hint: 'No bridge-local job or remote project matched. Call list_projects to inspect available projects.',
+                };
+            }
+            const status = project.status || (project.chunk_count > 0 ? 'ready' : 'pending');
+            return {
+                project_id: project.id || project.project_id,
+                project_name: project.name,
+                status,
+                files_indexed: project.file_count,
+                chunks_created: project.chunk_count,
+                languages: project.languages,
+                last_indexed_at: project.last_indexed_at,
+                updated_at: project.updated_at,
+                hint: status === 'indexing'
+                    ? 'Project is still indexing. If index_project returned bridge_job_id, poll get_indexing_status with that id for precise bridge progress.'
+                    : 'Project status loaded from list_projects.',
+            };
+        } catch (error) {
+            return {
+                error: 'status_lookup_failed',
+                project_id: projectIdOrName,
+                hint: error && error.message ? error.message : 'Could not parse list_projects response.',
+            };
+        }
+    }
+
     /**
      * Compare each result's indexed_file_hash against the local file's
      * current SHA-256 and set a `stale` field if they diverge. Works for

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cf-memory-mcp",
-	"version": "3.9.8",
+	"version": "3.9.11",
 	"description": "Cloudflare-hosted MCP server for code indexing, retrieval, and assistant memory with a direct remote MCP endpoint and local stdio bridge.",
 	"main": "bin/cf-memory-mcp.js",
 	"bin": {