cf-memory-mcp 3.8.5 → 3.8.7

package/bin/cf-memory-mcp.js

@@ -1,5 +1,34 @@
 #!/usr/bin/env node
 
+// Diagnostic logging — captures every MCP request/response with timing to
+// /tmp/cf-memory-mcp.log when CF_MEMORY_TRACE=1 is set. Helps debug "tool
+// gets stuck" reports without bothering users running normally.
+// Uses globalThis.process to avoid TDZ with the later `const process = require('process')`.
+const _MCP_TRACE = globalThis.process.env.CF_MEMORY_TRACE === '1' || globalThis.process.env.CF_MEMORY_TRACE === 'true';
+const _MCP_TRACE_PATH = '/tmp/cf-memory-mcp.log';
+const _MCP_TRACE_MAX_BYTES = 5 * 1024 * 1024; // 5MB cap; rotates by truncating
+
+function _mcpTrace(label, data) {
+    if (!_MCP_TRACE) return;
+    try {
+        const fs = require('fs');
+        // Cheap rotation: if the file is too big, truncate it. We don't
+        // archive — debugging old sessions isn't important enough to
+        // justify managing rotation files.
+        try {
+            const st = fs.statSync(_MCP_TRACE_PATH);
+            if (st.size > _MCP_TRACE_MAX_BYTES) {
+                fs.truncateSync(_MCP_TRACE_PATH, 0);
+                fs.appendFileSync(_MCP_TRACE_PATH,
+                    `[${new Date().toISOString()}] [pid=${globalThis.process.pid}] TRUNCATE: log exceeded ${_MCP_TRACE_MAX_BYTES} bytes\n`);
+            }
+        } catch (_) { /* file doesn't exist yet, that's fine */ }
+        fs.appendFileSync(_MCP_TRACE_PATH,
+            `[${new Date().toISOString()}] [pid=${globalThis.process.pid}] ${label}: ${data}\n`);
+    } catch (_) {}
+}
+_mcpTrace('STARTUP', `node=${globalThis.process.version} args=${JSON.stringify(globalThis.process.argv.slice(2))}`);
+
 /**
  * CF Memory MCP - Portable MCP Server
  * 
@@ -22,19 +51,312 @@ const path = require('path');
 const crypto = require('crypto');
 
 // Configuration
-const STREAMABLE_HTTP_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
-const LEGACY_SERVER_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/mcp/message';
-const PROGRESS_SSE_URL = 'https://cf-memory-mcp-simplified.johnlam90.workers.dev/api/indexing/progress';
+// Migrate users with stale shell exports pointing to the dead workers.dev URL.
+// This was the old default; if it's set in the user's shell env it would
+// silently override the .mcp.json setting and make every tool call fail.
+function _resolveBaseUrl() {
+    const raw = process.env.CF_MEMORY_BASE_URL;
+    if (!raw || raw.includes('workers.dev')) {
+        return 'https://memcp.ai';
+    }
+    return raw;
+}
+const BASE_URL = _resolveBaseUrl();
+const STREAMABLE_HTTP_URL = `${BASE_URL}/mcp`;
+const LEGACY_SERVER_URL = `${BASE_URL}/mcp/message`;
+const PROGRESS_SSE_URL = `${BASE_URL}/api/indexing/progress`;
 const PACKAGE_VERSION = require('../package.json').version;
-const TIMEOUT_MS = 60000; // Increased timeout for batch operations
+// Default per-request timeout. Batch uploads use BATCH_TIMEOUT_MS below.
+const TIMEOUT_MS = 60000;
+// Batch uploads can take longer because the worker processes each file
+// (hash, parse, embed, store). Give them a wider window.
+const BATCH_TIMEOUT_MS = Number(process.env.CF_MEMORY_BATCH_TIMEOUT_MS || 180000);
 const CONNECT_TIMEOUT_MS = 10000;
 
+// HTTPS agent with keep-alive to reuse TLS connections across requests.
+// Reduces latency by avoiding repeated TLS handshakes.
+const httpsAgent = new https.Agent({
+    keepAlive: true,
+    keepAliveMsecs: 30000,
+    maxSockets: 10,
+    maxFreeSockets: 5,
+    timeout: TIMEOUT_MS,
+});
+
 // Get API key from environment variable (will be checked later)
 const API_KEY = process.env.CF_MEMORY_API_KEY;
 
-// Optional: stream indexing progress via SSE
+// CF_MEMORY_PROGRESS=1 → stream indexing progress via SSE to stderr.
+// The bridge fires startProgressStream() with a generated session id when
+// a long-running index_project starts, and the server emits per-file
+// events through /api/indexing/progress.
 const ENABLE_PROGRESS = process.env.CF_MEMORY_PROGRESS === '1' || process.env.CF_MEMORY_PROGRESS === 'true';
 
+// Static tools list for fast local response (avoids network round-trip during connection)
+const TOOLS_LIST = [
+    {
+        name: 'index_project',
+        description: 'Index a local codebase for semantic code search. Scans files, parses into chunks (functions, classes, methods, types), generates embeddings, and stores them for retrieval. Use this BEFORE calling retrieve_context on a new project. Incremental: skips unchanged files based on content hash. Supports 110+ languages including TypeScript, Python, Go, Rust, Java, C++, Swift, Ruby, GLSL/HLSL shaders, and more.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_path: { type: 'string', description: 'Absolute path to the project root directory' },
+                project_name: { type: 'string', description: 'Display name for the project (defaults to directory basename)' },
+                force_reindex: { type: 'boolean', description: 'If true, wipes existing chunks and rebuilds from scratch. Use only when needed; incremental is much faster.' }
+            },
+            required: ['project_path']
+        }
+    },
+    {
+        name: 'index_github',
+        description: 'Index a public GitHub repository server-side without cloning locally. Use when the user wants to search a remote codebase.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                repo_url: { type: 'string', description: 'GitHub URL (e.g., https://github.com/user/repo)' },
+                branch: { type: 'string', description: 'Branch to index (default: main)' }
+            },
+            required: ['repo_url']
+        }
+    },
+    {
+        name: 'list_projects',
+        description: 'List all indexed projects with their stats (file count, chunk count, languages, last indexed time). Use to discover what is available for retrieval before searching.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                page: { type: 'number', description: 'Page number for pagination' },
+                limit: { type: 'number', description: 'Max projects per page' }
+            }
+        }
+    },
+    {
+        name: 'delete_project',
+        description: 'Permanently delete an indexed project and all its chunks/relationships. Cannot be undone.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID (proj_xxx) to delete' }
+            },
+            required: ['project_id']
+        }
+    },
+    {
+        name: 'retrieve_context',
+        description: 'Search indexed code using hybrid retrieval (semantic embeddings + keyword BM25 + identifier name matching), with reciprocal rank fusion and cross-encoder reranking. Returns chunks enriched with file_imports, source_kind (code/test/doc/config), indexed_at, and a `stale` flag when the file has been edited since indexing. Doc/markdown filtered out of code queries by default. Use for any code search: "where is X defined", "how does Y work", "what handles Z".',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'Natural language query or specific identifier name' },
+                project_id: { type: 'string', description: 'Project ID (proj_xxx) or project name. Omit to search most recently used project.' },
+                limit: { type: 'number', description: 'Max results to return (default: 10, max: 50)' },
+                language_filter: { type: 'array', items: { type: 'string' }, description: 'Limit to languages, e.g. ["typescript", "python"]' },
+                chunk_type_filter: { type: 'array', items: { type: 'string' }, description: 'Limit to chunk types, e.g. ["function", "class", "interface"]' },
+                file_filter: { type: 'array', items: { type: 'string' }, description: 'Limit to files matching path substrings' },
+                all_projects: { type: 'boolean', description: 'Search across ALL your indexed projects (overrides project_id). Useful for "find X in any of my repos".' },
+                expand_context: { type: 'boolean', description: 'Include file_imports (the file\'s module/imports chunk) with each result. Default: true.' },
+                exclude_docs: { type: 'boolean', description: 'Filter out markdown/docs from code queries. Default: true (auto-disabled if query mentions docs/readme/tutorial).' }
+            },
+            required: ['query']
+        }
+    },
+    {
+        name: 'store_memory',
+        description: 'Persist a memory across conversations (fact, preference, task, entity, or session summary). Use for things the user explicitly wants remembered: their preferences, important facts about projects, ongoing tasks.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                content: { type: 'string', description: 'The memory content to store' },
+                type: { type: 'string', enum: ['fact', 'preference', 'task', 'entity', 'session_summary'] },
+                importance: { type: 'number', minimum: 0, maximum: 1, description: 'Importance score 0-1; higher = more likely to surface in retrieval' }
+            },
+            required: ['content']
+        }
+    },
+    {
+        name: 'retrieve_memories',
+        description: 'Semantic search across stored memories. Use to recall what the user previously shared or asked about.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'Natural language query' },
+                limit: { type: 'number', description: 'Max memories to return' }
+            },
+            required: ['query']
+        }
+    },
+    {
+        name: 'get_context_bootstrap',
+        description: 'Get the most important memories to load at session start. Returns user preferences, ongoing tasks, and relevant facts within the token budget.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                max_tokens: { type: 'number', description: 'Token budget for bootstrap context (default: 1000)' },
+                current_context: { type: 'string', description: 'Current conversation context to bias relevance' }
+            }
+        }
+    },
+    {
+        name: 'start_session',
+        description: 'Begin a new tracked conversation session. Returns a session_id used by end_session for summarization.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                context: { type: 'string', enum: ['main', 'group', 'background'], description: 'Session context type' },
+                platform: { type: 'string', description: 'Client platform (e.g., "claude-code", "claude-desktop")' }
+            },
+            required: ['context']
+        }
+    },
+    {
+        name: 'end_session',
+        description: 'End a tracked session and optionally extract memories from the summary.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                session_id: { type: 'string', description: 'Session ID returned by start_session' },
+                summary: { type: 'string', description: 'Optional summary of what was discussed' }
+            },
+            required: ['session_id']
+        }
+    },
+    {
+        name: 'store_entity',
+        description: 'Store a structured entity (person, project, company, concept, location) with named attributes for later relational queries.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                name: { type: 'string', description: 'Entity name' },
+                type: { type: 'string', enum: ['person', 'project', 'company', 'concept', 'location'] },
+                attributes: { type: 'object', description: 'Key-value attributes of the entity' }
+            },
+            required: ['name', 'type', 'attributes']
+        }
+    },
+    {
+        name: 'get_related_code',
+        description: 'Navigate the code relationship graph: find callers (who calls this), callees (what this calls), imports, type usages, and inheritance for a specific chunk. Use after retrieve_context to explore connected code.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID' },
+                chunk_name: { type: 'string', description: 'Name of the function/class/method (looked up by name)' },
+                chunk_id: { type: 'string', description: 'Specific chunk ID (alternative to chunk_name)' },
+                relationship_type: { type: 'string', enum: ['calls', 'imports', 'extends', 'implements', 'uses_type', 'all'], description: 'Type of relationship (default: all)' },
+                direction: { type: 'string', enum: ['callers', 'callees', 'both'], description: 'Direction of edges to traverse (default: both)' },
+                limit: { type: 'number', description: 'Max related chunks per direction (default: 20)' }
+            },
+            required: ['project_id']
+        }
+    },
+    {
+        name: 'get_stats',
+        description: 'Get aggregated statistics across all your indexed projects: total project/file/chunk counts plus language breakdown. Use to understand what is available before searching, or to verify indexing succeeded.',
+        inputSchema: {
+            type: 'object',
+            properties: {}
+        }
+    },
+    {
+        name: 'list_files',
+        description: 'List indexed files in a project, ranked by chunk count (most complex files first). Filter by path substring or language. Use to explore project structure before searching.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                path_pattern: { type: 'string', description: 'Filter files by path substring (e.g., "services/" or "utils.ts")' },
+                language: { type: 'string', description: 'Filter by language' },
+                limit: { type: 'number', description: 'Max files (default: 100, max: 500)' }
+            },
+            required: ['project_id']
+        }
+    },
+    {
+        name: 'get_file_outline',
+        description: 'Get the structural outline of a file: functions, classes, methods, interfaces with their line ranges. Returns names only (not content) for a quick overview. Supports partial file path matching.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                file_path: { type: 'string', description: 'File path (exact or partial substring match)' }
+            },
+            required: ['project_id', 'file_path']
+        }
+    },
+    {
+        name: 'get_file_content',
+        description: 'Reassemble and return the full content of an indexed file from its stored chunks. Use when retrieve_context returned a fragment and you need the whole file context, or when working remotely without local filesystem access. Returns indexed_at + file_hash for staleness checks.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                file_path: { type: 'string', description: 'File path (exact or partial substring match)' },
+                max_chars: { type: 'number', description: 'Truncate content at this many characters (default 50000, max 100000)' }
+            },
+            required: ['project_id', 'file_path']
+        }
+    },
+    {
+        name: 'health_check',
+        description: 'Verify the MCP server is reachable and authentication works. Returns the user_id, server name/version, and a current timestamp. Use as a first probe when troubleshooting connectivity.',
+        inputSchema: {
+            type: 'object',
+            properties: {}
+        }
+    },
+    {
+        name: 'refresh_files',
+        description: 'Re-index specific files without scanning the whole project. Use after retrieve_context returns stale results — pass the file paths from the stale chunks and only those files get re-embedded. Much faster than a full re-index for targeted updates.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                file_paths: { type: 'array', items: { type: 'string' }, description: 'Paths relative to project root (e.g. ["src/foo.ts", "src/bar.ts"])' },
+                project_root: { type: 'string', description: 'Absolute path to project root. Defaults to CF_MEMORY_WATCH_PATH or current working directory.' }
+            },
+            required: ['project_id', 'file_paths']
+        }
+    },
+    {
+        name: 'find_stale_files',
+        description: 'Compare indexed file hashes against current local file hashes. Returns lists of stale (edited since indexing), missing (file removed locally), and fresh files. Includes a hint to refresh_files for the stale ones. Use at session start to know what needs re-indexing.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                project_root: { type: 'string', description: 'Absolute path to project root. Defaults to CF_MEMORY_WATCH_PATH or cwd.' },
+                limit: { type: 'number', description: 'Max files to inspect (default 500)' }
+            },
+            required: ['project_id']
+        }
+    },
+    {
+        name: 'refresh_stale',
+        description: 'One-call convenience: finds stale files (edited since indexing) and refreshes them in one operation. Use this at session start or whenever you suspect the index might be out of date.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                project_id: { type: 'string', description: 'Project ID or name' },
+                project_root: { type: 'string', description: 'Absolute path to project root. Defaults to CF_MEMORY_WATCH_PATH or cwd.' },
+                max_files: { type: 'number', description: 'Cap how many stale files get refreshed in one call (default 100)' }
+            },
+            required: ['project_id']
+        }
+    },
+    {
+        name: 'delete_memory',
+        description: 'Delete persisted memories. Requires at least one filter (memory_id, type, or older_than_days) to prevent accidental mass deletion. Use to clean up outdated memories or remove obsolete preferences.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                memory_id: { type: 'string', description: 'Specific memory ID to delete' },
+                type: { type: 'string', enum: ['fact', 'preference', 'task', 'entity', 'session_summary'], description: 'Delete all memories of this type' },
+                older_than_days: { type: 'number', description: 'Delete memories created more than N days ago' }
+            }
+        }
+    }
+];
+
 /**
  * Cross-platform MCP stdio bridge
  * Handles communication between MCP clients and the Cloudflare Worker
@@ -57,7 +379,7 @@ class CFMemoryMCP {
 
         // Set up stdio encoding
         process.stdin.setEncoding('utf8');
-        process.stdout.setEncoding('utf8');
+        // Note: stdout.setEncoding doesn't exist on writable streams
 
         this.logDebug('CF Memory MCP server starting...');
         this.logDebug(`Streamable HTTP URL: ${this.streamableHttpUrl}`);
@@ -90,20 +412,80 @@ class CFMemoryMCP {
      */
     async start() {
         try {
-            // Skip connectivity test in MCP mode - it will be tested when first request is made
             this.logDebug('Starting MCP message processing...');
-            
+
+            // Pre-warm the HTTPS connection in the background so the first
+            // real tool call doesn't pay the TLS handshake cost.
+            this.prewarmConnection();
+
+            // Background-prime the project_id auto-detection cache by calling
+            // list_projects and matching against cwd. By the time the first
+            // retrieve_context arrives, the cache is hot — no extra roundtrip.
+            this.prewarmProjectIdCache();
+
             // Start auto-watcher if CF_MEMORY_AUTO_WATCH is set
             if (process.env.CF_MEMORY_AUTO_WATCH === '1' || process.env.CF_MEMORY_AUTO_WATCH === 'true') {
                 this.startAutoWatcher();
             }
-            
+
             await this.processStdio();
         } catch (error) {
             this.logError('Failed to start MCP server:', error);
             process.exit(1);
         }
     }
+
+    /**
+     * Resolve the current cwd to a project_id in the background and
+     * cache the result, so the first retrieve_context query that
+     * triggers maybeFillProjectId() gets an instant hit.
+     */
+    async prewarmProjectIdCache() {
+        try {
+            // Fake message just to drive the helper through its happy path.
+            // It'll call list_projects, find the matching project for cwd,
+            // and populate this._projectIdByCwdCache.
+            const fakeMessage = {
+                params: {
+                    name: 'retrieve_context',
+                    arguments: {}, // no project_id — triggers auto-fill
+                },
+            };
+            await this.maybeFillProjectId(fakeMessage);
+        } catch (err) {
+            this.logDebug(`prewarmProjectIdCache failed: ${err && err.message}`);
+        }
+    }
+
+    /**
+     * Pre-warm the HTTPS connection so the first real request is fast.
+     * Fires a lightweight HEAD-equivalent (initialize) in the background.
+     * Failure is silently ignored - this is just an optimization.
+     */
+    prewarmConnection() {
+        const url = new URL(BASE_URL + '/health');
+        const options = {
+            hostname: url.hostname,
+            port: url.port || 443,
+            path: url.pathname,
+            method: 'GET',
+            timeout: CONNECT_TIMEOUT_MS,
+            agent: httpsAgent,
+            headers: { 'User-Agent': this.userAgent }
+        };
+
+        try {
+            const req = https.request(options, (res) => {
+                res.on('data', () => {});
+                res.on('end', () => this.logDebug('Connection pre-warmed'));
+            });
+            req.on('error', () => {});
+            req.on('timeout', () => req.destroy());
+            req.end();
+        } catch (_) {
+            // Ignore pre-warm failures
+        }
+    }
     
     /**
      * Start auto-watching files for changes
@@ -147,7 +529,7 @@ class CFMemoryMCP {
                 protocolVersion: '2025-03-26',
                 capabilities: { tools: {} },
                 clientInfo: {
-                    name: 'cf-memory-mcp',
+                    name: 'cf-memory-mcp-simplified',
                     version: PACKAGE_VERSION
                 }
             }
@@ -180,52 +562,70 @@ class CFMemoryMCP {
      * Process stdio input/output for MCP communication
      */
     async processStdio() {
-        let buffer = '';
-
-        // Handle stdin data
-        for await (const chunk of process.stdin) {
-            buffer += chunk;
+        const readline = require('readline');
 
-            // Process complete JSON-RPC messages (one per line)
-            let newlineIndex;
-            while ((newlineIndex = buffer.indexOf('\n')) !== -1) {
-                const line = buffer.slice(0, newlineIndex).trim();
-                buffer = buffer.slice(newlineIndex + 1);
+        const rl = readline.createInterface({
+            input: process.stdin,
+            terminal: false
+        });
 
-                if (line) {
-                    await this.handleMessage(line);
-                }
-            }
-        }
+        // Process each line concurrently. JSON-RPC messages have unique ids,
+        // so responses can be matched even if they arrive out of order. This
+        // prevents a slow tool call (e.g., a remote retrieve) from blocking
+        // all subsequent calls, which the user reported as "MCP gets stuck".
+        rl.on('line', (line) => {
+            const trimmed = line.trim();
+            if (!trimmed) return;
+            // Fire-and-forget. handleMessage catches its own errors and writes
+            // a JSON-RPC error response, so unhandled rejections shouldn't escape.
+            this.handleMessage(trimmed).catch(err => {
+                this.logError('Unhandled message handler error:', err);
+            });
+        });
 
-        // Process any remaining buffer content
-        if (buffer.trim()) {
-            await this.handleMessage(buffer.trim());
-        }
+        // Handle stdin close
+        rl.on('close', () => {
+            this.logDebug('Stdin closed, shutting down...');
+            process.exit(0);
+        });
 
-        this.logDebug('Stdin closed, shutting down...');
+        // Keep the process alive
+        await new Promise(() => {});
     }
 
     /**
      * Handle a single MCP message
      */
     async handleMessage(messageStr) {
+        const _t0 = Date.now();
+        _mcpTrace('IN', messageStr.length > 500 ? messageStr.slice(0, 500) + '...' : messageStr);
         try {
             const message = JSON.parse(messageStr);
             this.logDebug(`Processing message: ${message.method} (id: ${message.id})`);
+            // Trace on early-exit paths handled below: returns after stdout.write.
+            // For brevity, we instead trace once at the end of every dispatch.
 
             // Handle lifecycle methods locally
             if (message.method === 'initialize') {
+                // Per MCP spec: echo the client's protocolVersion if we support
+                // it, otherwise return our preferred version. We support both
+                // the 2024-11-05 and 2025-03-26 wire formats.
+                const SUPPORTED_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05'];
+                const PREFERRED_VERSION = '2025-03-26';
+                const clientVersion = message.params && message.params.protocolVersion;
+                const respondVersion = SUPPORTED_VERSIONS.includes(clientVersion)
+                    ? clientVersion
+                    : PREFERRED_VERSION;
                 const response = {
                     jsonrpc: '2.0',
                     id: message.id,
                     result: {
-                        protocolVersion: '2025-03-26',
+                        protocolVersion: respondVersion,
                         capabilities: {
                             tools: {}
                         },
                         serverInfo: {
-                            name: 'cf-memory-mcp',
+                            name: 'cf-memory-mcp-simplified',
                             version: PACKAGE_VERSION
                         }
                     }
@@ -239,24 +639,131 @@ class CFMemoryMCP {
                 return;
             }
 
+            // Handle tools/list locally for fast connection
+            if (message.method === 'tools/list') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: { tools: TOOLS_LIST }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // Handle resources/list locally (we have none)
+            if (message.method === 'resources/list') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: { resources: [] }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // Handle resources/templates/list locally (we have none)
+            if (message.method === 'resources/templates/list') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: { resourceTemplates: [] }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // Handle prompts/list locally (we have none)
+            if (message.method === 'prompts/list') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: { prompts: [] }
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // Handle ping locally
+            if (message.method === 'ping') {
+                const response = {
+                    jsonrpc: '2.0',
+                    id: message.id,
+                    result: {}
+                };
+                process.stdout.write(JSON.stringify(response) + '\n');
+                return;
+            }
+
+            // Intercept refresh_files: read specific files locally and upload
+            // them to refresh the index without re-scanning the whole project.
+            if (message.method === 'tools/call' && message.params && message.params.name === 'refresh_files') {
+                await this.handleRefreshFiles(message);
+                return;
+            }
+
+            // Intercept find_stale_files: compare indexed file hashes against
+            // current local file hashes to identify what needs refresh.
+            if (message.method === 'tools/call' && message.params && message.params.name === 'find_stale_files') {
+                await this.handleFindStaleFiles(message);
+                return;
+            }
+
+            // Intercept refresh_stale: find_stale_files + refresh_files in one call.
+            if (message.method === 'tools/call' && message.params && message.params.name === 'refresh_stale') {
+                await this.handleRefreshStale(message);
+                return;
+            }
+
             // Intercept index_project tool call to perform local scanning
             if (message.method === 'tools/call' && message.params && message.params.name === 'index_project') {
                 await this.handleIndexProject(message);
                 return;
             }
 
+            _mcpTrace('DISPATCH', `id=${message.id} method=${message.method} -> network`);
+            // If this is a retrieve_context call with no project_id and no
+            // all_projects, try to auto-fill project_id from the current
+            // working directory. Saves the user from having to specify
+            // project_id every time when working in a known project.
+            if (message.method === 'tools/call' &&
+                message.params && message.params.name === 'retrieve_context') {
+                await this.maybeFillProjectId(message);
+            }
+
             const response = await this.makeRequest(message);
+            _mcpTrace('DISPATCH_DONE', `id=${message.id} method=${message.method} elapsed=${Date.now()-_t0}ms`);
+
+            // Annotate retrieve_context results with local staleness when
+            // possible. The server tells us when it indexed each file and
+            // what hash; we compare against the local file. Stops users
+            // acting on stale indexed content as if it were current.
+            if (message.method === 'tools/call' &&
+                message.params && message.params.name === 'retrieve_context') {
+                this.maybeAnnotateStaleness(response, message.params.arguments);
+            }
 
             // Send response to stdout
             process.stdout.write(JSON.stringify(response) + '\n');
 
         } catch (error) {
             this.logError('Error handling message:', error);
+            _mcpTrace('ERROR', `${error.message} elapsed=${Date.now()-_t0}ms`);
+
+            // Try to extract message id so the client can correlate the error.
+            // Without an id, the client may wait indefinitely for a response.
+            let messageId = null;
+            try {
+                const parsed = JSON.parse(messageStr);
+                if (parsed && parsed.id !== undefined) {
+                    messageId = parsed.id;
+                }
+            } catch (_) {
+                // messageStr is unparseable; nothing to correlate.
+            }
 
-            // Send error response
             const errorResponse = {
                 jsonrpc: '2.0',
-                id: null,
+                id: messageId,
                 error: {
                     code: -32700,
                     message: 'Parse error',
@@ -270,6 +777,285 @@ class CFMemoryMCP {
     /**
      * Handle local project indexing - scans local files and sends them via MCP
      */
+    /**
+     * Refresh specific files in the index without re-scanning the whole
+     * project. Reads each file locally and uploads to /api/projects/:id/files/batch.
+     * Use this after retrieve_context returns stale results — instead of doing
+     * a full re-index, just refresh the affected files.
+     */
+    async handleRefreshFiles(message) {
+        const args = (message.params && message.params.arguments) || {};
+        const projectIdOrName = args.project_id;
+        const filePaths = Array.isArray(args.file_paths) ? args.file_paths : [];
+        const projectRoot = args.project_root ? path.resolve(args.project_root)
+            : (process.env.CF_MEMORY_WATCH_PATH || process.cwd());
+
+        const respond = (payload) => {
+            process.stdout.write(JSON.stringify({
+                jsonrpc: '2.0',
+                id: message.id,
+                result: { content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }] },
+            }) + '\n');
+        };
+
+        if (!projectIdOrName) {
+            return respond({ error: 'project_id is required' });
+        }
+        if (filePaths.length === 0) {
+            return respond({ error: 'file_paths (string[]) is required' });
+        }
+
+        // Resolve to project ID via list_projects if a name was given.
+        let projectId = projectIdOrName;
+        if (!projectId.startsWith('proj_')) {
+            const list = await this.makeRequest({
+                jsonrpc: '2.0',
+                id: `refresh-list-${Date.now()}`,
+                method: 'tools/call',
+                params: { name: 'list_projects', arguments: {} },
+            });
+            try {
+                const projects = JSON.parse(list.result.content[0].text);
+                const match = Array.isArray(projects)
+                    ? projects.find(p => p.name === projectId || p.id === projectId)
+                    : null;
+                if (match?.id) projectId = match.id;
+            } catch (_) {}
+        }
+
+        // Read each file locally. Note: uploadFileBatch expects `relativePath`
+        // (not `path`) — it maps to `file_path` server-side.
+        const files = [];
+        const skipped = [];
+        for (const rel of filePaths) {
+            try {
+                const full = path.resolve(projectRoot, rel);
+                const stat = fs.statSync(full);
+                if (!stat.isFile()) {
+                    skipped.push({ path: rel, reason: 'not a file' });
+                    continue;
+                }
+                const content = fs.readFileSync(full, 'utf8');
+                files.push({
+                    relativePath: rel,
+                    content,
+                    last_modified: stat.mtime.toISOString(),
+                });
+            } catch (err) {
+                skipped.push({ path: rel, reason: err && err.message ? err.message : 'read failed' });
+            }
+        }
+
+        if (files.length === 0) {
+            return respond({
+                project_id: projectId,
+                files_refreshed: 0,
+                skipped,
+            });
+        }
+
+        const uploadResult = await this.uploadFileBatch(projectId, files);
+        const refreshed = (uploadResult && typeof uploadResult.files_indexed === 'number') ? uploadResult.files_indexed : 0;
+        const chunks = (uploadResult && typeof uploadResult.chunks_created === 'number') ? uploadResult.chunks_created : 0;
+
+        return respond({
+            project_id: projectId,
+            files_attempted: files.length,
+            files_refreshed: refreshed,
+            chunks_created: chunks,
+            skipped,
+            errors: uploadResult && Array.isArray(uploadResult.errors) ? uploadResult.errors : undefined,
+        });
+    }
+
+    /**
+     * Check which indexed files have been edited locally since their
+     * last index. Calls list_files to get the indexed files + their
+     * hashes, then reads each local file and compares SHA-256.
+     */
+    async handleFindStaleFiles(message) {
+        const args = (message.params && message.params.arguments) || {};
+        const projectIdOrName = args.project_id;
+        const projectRoot = args.project_root ? path.resolve(args.project_root)
+            : (process.env.CF_MEMORY_WATCH_PATH || process.cwd());
+        const limit = args.limit || 500;
+
+        const respond = (payload) => {
+            process.stdout.write(JSON.stringify({
+                jsonrpc: '2.0',
+                id: message.id,
+                result: { content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }] },
+            }) + '\n');
+        };
+
+        if (!projectIdOrName) return respond({ error: 'project_id is required' });
+
+        // Get the list of indexed files with their hashes via the new
+        // /api/projects/:id/files endpoint... actually list_files returns
+        // metadata but not file_hash. We'll add hash to the response.
+        // For now, fetch files via direct API.
+        const listFilesRes = await this.makeRequest({
+            jsonrpc: '2.0',
+            id: `stale-list-${Date.now()}`,
+            method: 'tools/call',
+            params: { name: 'list_files', arguments: { project_id: projectIdOrName, limit } },
+        });
+
+        let filesList = [];
+        try {
+            filesList = JSON.parse(listFilesRes.result.content[0].text);
+            if (!Array.isArray(filesList)) filesList = [];
+        } catch (_) {
+            return respond({ error: 'Failed to list indexed files' });
+        }
+
+        // Each file in the list already carries file_hash + indexed_at
+        // (list_files was extended to return them), so we don't need any
+        // additional server roundtrips. Just hash the local files.
+        const stale = [];
+        const missing = [];
+        const fresh = [];
+        for (const f of filesList) {
+            const filePath = f.file_path;
+            const indexedHash = f.file_hash;
+            const indexedAt = f.indexed_at;
+            if (!indexedHash) continue;
+
+            const full = path.resolve(projectRoot, filePath);
+            let content;
+            try {
+                content = fs.readFileSync(full, 'utf8');
+            } catch (_) {
+                missing.push({ file_path: filePath, indexed_at: indexedAt, reason: 'file missing locally' });
+                continue;
+            }
+            const localHash = crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+            if (localHash !== indexedHash) {
+                stale.push({ file_path: filePath, indexed_at: indexedAt });
+            } else {
+                fresh.push(filePath);
+            }
+        }
+
+        return respond({
+            project_id: projectIdOrName,
+            total_indexed: filesList.length,
+            fresh_count: fresh.length,
+            stale_count: stale.length,
+            missing_count: missing.length,
+            stale,
+            missing,
+            hint: stale.length > 0
+                ? `Call refresh_files with file_paths=${JSON.stringify(stale.map(s => s.file_path).slice(0, 10))} to update the index.`
+                : 'Index is up to date.',
+        });
+    }
+
+    /**
+     * Convenience: find stale files and refresh them in one call.
+     * Equivalent to find_stale_files + refresh_files but with one
+     * round-trip instead of two and no need to copy paths between calls.
+     */
+    async handleRefreshStale(message) {
+        const args = (message.params && message.params.arguments) || {};
+        const projectIdOrName = args.project_id;
+        const projectRoot = args.project_root ? path.resolve(args.project_root)
+            : (process.env.CF_MEMORY_WATCH_PATH || process.cwd());
+        const maxFiles = args.max_files || 100;
+
+        const respond = (payload) => {
+            process.stdout.write(JSON.stringify({
+                jsonrpc: '2.0',
+                id: message.id,
+                result: { content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }] },
+            }) + '\n');
+        };
+
+        if (!projectIdOrName) return respond({ error: 'project_id is required' });
+
+        // Step 1: find stale files (reuses the existing handler logic).
+        // We can't call handleFindStaleFiles directly because it writes to
+        // stdout, so duplicate the small core.
+        const listRes = await this.makeRequest({
+            jsonrpc: '2.0',
+            id: `rs-list-${Date.now()}`,
+            method: 'tools/call',
+            params: { name: 'list_files', arguments: { project_id: projectIdOrName, limit: 500 } },
+        });
+        let filesList = [];
+        try {
+            filesList = JSON.parse(listRes.result.content[0].text);
+            if (!Array.isArray(filesList)) filesList = [];
+        } catch (_) {
+            return respond({ error: 'Failed to list indexed files' });
+        }
+
+        const staleFiles = [];
+        const missing = [];
+        for (const f of filesList) {
+            const filePath = f.file_path;
+            const indexedHash = f.file_hash;
+            if (!indexedHash) continue;
+            const full = path.resolve(projectRoot, filePath);
+            let content;
+            try {
+                content = fs.readFileSync(full, 'utf8');
+            } catch (_) {
+                missing.push(filePath);
+                continue;
+            }
+            const localHash = crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+            if (localHash !== indexedHash) {
+                staleFiles.push({ relativePath: filePath, content, last_modified: '' });
+            }
+        }
+
+        const toRefresh = staleFiles.slice(0, maxFiles);
+        if (toRefresh.length === 0) {
+            return respond({
+                project_id: projectIdOrName,
+                stale_count: 0,
+                missing_count: missing.length,
+                refreshed: 0,
+                message: 'Index is up to date.',
+            });
+        }
+
+        // Resolve project name to ID once
+        let projectId = projectIdOrName;
+        if (!projectId.startsWith('proj_') && filesList.length > 0) {
+            // list_files only works if we can resolve the project; we already
+            // got results so just look up via list_projects to get the ID.
+            const projListRes = await this.makeRequest({
+                jsonrpc: '2.0',
+                id: `rs-proj-${Date.now()}`,
+                method: 'tools/call',
+                params: { name: 'list_projects', arguments: {} },
+            });
+            try {
+                const projects = JSON.parse(projListRes.result.content[0].text);
+                const match = Array.isArray(projects)
+                    ? projects.find(p => p.name === projectId || p.id === projectId)
+                    : null;
+                if (match?.id) projectId = match.id;
+            } catch (_) {}
+        }
+
+        const uploadResult = await this.uploadFileBatch(projectId, toRefresh);
+        const refreshed = (uploadResult && typeof uploadResult.files_indexed === 'number') ? uploadResult.files_indexed : 0;
+        const chunks = (uploadResult && typeof uploadResult.chunks_created === 'number') ? uploadResult.chunks_created : 0;
+
+        return respond({
+            project_id: projectId,
+            stale_count: staleFiles.length,
+            missing_count: missing.length,
+            refreshed,
+            chunks_created: chunks,
+            refreshed_files: toRefresh.map(f => f.relativePath),
+            truncated: staleFiles.length > maxFiles,
+        });
+    }
+
     async handleIndexProject(message) {
         const { project_path, project_name, include_patterns, exclude_patterns, force_reindex } = message.params.arguments;
         const resolvedPath = path.resolve(project_path);
@@ -277,6 +1063,20 @@ class CFMemoryMCP {
 
         this.logDebug(`Intercepted index_project for: ${resolvedPath} (${name})`);
 
+        // Optionally start a progress stream so the user sees per-file events
+        // on stderr as indexing happens. Only useful for long indexes; cheap
+        // to skip when unset (the env check is at module load).
+        let progressStream = null;
+        if (ENABLE_PROGRESS) {
+            const sessionId = `idx-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+            try {
+                progressStream = this.startProgressStream(sessionId);
+                process.stderr.write(`[INDEX] streaming progress (session=${sessionId})\n`);
+            } catch (err) {
+                this.logDebug(`Could not start progress stream: ${err && err.message}`);
+            }
+        }
+
         try {
             // 1. Scan Local Files
             this.logDebug(`Scanning files in ${resolvedPath}...`);
@@ -344,19 +1144,53 @@ class CFMemoryMCP {
             // Adaptive batching: limit by file count and payload bytes to reduce timeouts.
             const batches = this.createAdaptiveBatches(files);
 
-            for (let b = 0; b < batches.length; b++) {
-                const batch = batches[b];
-                const approxBytes = batch.reduce((sum, f) => sum + Buffer.byteLength(f.content || '', 'utf8'), 0);
-                this.logDebug(`Uploading batch ${b + 1}/${batches.length} (${batch.length} files, ~${Math.round(approxBytes / 1024)}KB)`);
-
-                const uploadResult = await this.uploadFileBatch(projectId, batch);
-
-                if (uploadResult && typeof uploadResult.files_indexed === 'number') {
+            let totalSkipped = 0;
+            let totalUnchanged = 0;
+            const batchErrors = [];
+            const failedBatches = [];
+
+            // Process batches in parallel (concurrency = 3). Each batch is
+            // independent on the server side, so overlapping them gives a
+            // ~3x speedup for large projects. Higher concurrency risks
+            // overwhelming the Cloudflare Worker / hitting per-account
+            // request limits, so we cap conservatively.
+            const CONCURRENCY = Math.min(3, batches.length);
+            const aggregateBatchResult = (uploadResult, b, batch) => {
+                if (!uploadResult) {
+                    failedBatches.push({ batch: b + 1, files: batch.length, reason: 'no response (timeout/network)' });
+                    this.logError(`Batch ${b + 1}/${batches.length} returned no response (timeout or network error). ${batch.length} files unaccounted for.`);
+                    return;
+                }
+                if (typeof uploadResult.files_indexed === 'number') {
                     totalIndexed += uploadResult.files_indexed;
                 }
-                if (uploadResult && typeof uploadResult.chunks_created === 'number') {
+                if (typeof uploadResult.chunks_created === 'number') {
                     totalChunks += uploadResult.chunks_created;
                 }
+                if (typeof uploadResult.files_skipped === 'number') {
+                    totalSkipped += uploadResult.files_skipped;
+                }
+                if (typeof uploadResult.files_unchanged === 'number') {
+                    totalUnchanged += uploadResult.files_unchanged;
+                }
+                if (Array.isArray(uploadResult.errors) && uploadResult.errors.length > 0) {
+                    for (const err of uploadResult.errors) {
+                        batchErrors.push(err);
+                    }
+                }
+            };
+
+            for (let i = 0; i < batches.length; i += CONCURRENCY) {
+                const window = [];
+                for (let j = i; j < Math.min(i + CONCURRENCY, batches.length); j++) {
+                    const batch = batches[j];
+                    const approxBytes = batch.reduce((sum, f) => sum + Buffer.byteLength(f.content || '', 'utf8'), 0);
+                    this.logDebug(`Uploading batch ${j + 1}/${batches.length} (${batch.length} files, ~${Math.round(approxBytes / 1024)}KB)`);
+                    window.push(
+                        this.uploadFileBatch(projectId, batch).then(res => aggregateBatchResult(res, j, batch))
+                    );
+                }
+                await Promise.all(window);
             }
 
             // 3. Cleanup stale files (accuracy): remove server-side files not present locally.
@@ -369,21 +1203,32 @@ class CFMemoryMCP {
                 }
             }
 
-            // 4. Return aggregated success
+            // 4. Return aggregated success (with skipped/error visibility)
+            const status = failedBatches.length > 0 ? 'partial' : 'complete';
+            const responsePayload = {
+                project_id: projectId,
+                project_name: name,
+                files_found: files.length,
+                files_indexed: totalIndexed,
+                files_unchanged: totalUnchanged,
+                files_skipped: totalSkipped,
+                chunks_created: totalChunks,
+                status
+            };
+            if (batchErrors.length > 0) {
+                responsePayload.errors = batchErrors.slice(0, 50);
+                responsePayload.errors_truncated = batchErrors.length > 50;
+            }
+            if (failedBatches.length > 0) {
+                responsePayload.failed_batches = failedBatches;
+            }
             const response = {
                 jsonrpc: '2.0',
                 id: message.id,
                 result: {
                     content: [{
                         type: 'text',
-                        text: JSON.stringify({
-                            project_id: projectId,
-                            project_name: name,
-                            files_found: files.length,
-                            files_indexed: totalIndexed,
-                            chunks_created: totalChunks,
-                            status: 'complete'
-                        }, null, 2)
+                        text: JSON.stringify(responsePayload, null, 2)
                     }]
                 }
             };
@@ -400,6 +1245,11 @@ class CFMemoryMCP {
                 }
             };
             process.stdout.write(JSON.stringify(response) + '\n');
+        } finally {
+            // Close the SSE progress stream now that indexing is done.
+            if (progressStream && typeof progressStream.stop === 'function') {
+                try { progressStream.stop(); } catch (_) {}
+            }
         }
     }
 
@@ -424,17 +1274,47 @@ class CFMemoryMCP {
             // IDE/Editor history & settings
             '.history', '.vscode', '.idea', '.vs',
             // Cloudflare/tooling
-            '.wrangler', '.turbo', '.cache'
+            '.wrangler', '.turbo', '.cache',
+            // Legacy/archived rule directories that poison retrieval with
+            // stale architecture docs and example prompts.
+            '.augment', '.kiro', '.intent', '.husky', '.claude',
         ];
 
-        // Default file extensions to include
+        // Default file extensions to include - covers 110+ languages
+        // Must stay in sync with src-simplified/utils/index.ts DEFAULT_INCLUDE_EXTENSIONS
         const DEFAULT_INCLUDE_EXTS = [
+            // Mainstream
             '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
             '.py', '.rb', '.go', '.rs', '.java', '.kt', '.scala',
             '.cs', '.cpp', '.c', '.h', '.hpp', '.swift', '.php',
-            '.sql', '.sh', '.bash',
-            '.json', '.yaml', '.yml', '.toml',
-            '.html', '.css', '.scss', '.vue', '.svelte',
+            // Functional & systems
+            '.lua', '.ex', '.exs', '.hs', '.lhs', '.pl', '.pm',
+            '.groovy', '.gvy', '.gradle', '.r', '.R', '.dart',
+            '.ml', '.mli', '.fs', '.fsi', '.fsx',
+            '.clj', '.cljs', '.cljc', '.edn', '.jl',
+            '.tf', '.tfvars', '.hcl', '.zig', '.nim', '.nims', '.nimble',
+            '.cr', '.vv', '.d', '.di', '.erl', '.hrl',
+            // Legacy & specialized
+            '.sol', '.cob', '.cbl', '.cpy', '.asm', '.s', '.S',
+            '.proto', '.graphql', '.gql', '.mk',
+            '.scm', '.ss', '.rkt', '.lisp', '.lsp', '.cl', '.pro', '.P',
+            '.m', '.mat', '.f', '.f90', '.f95', '.f03', '.for',
+            '.adb', '.ads', '.ada', '.vhd', '.vhdl', '.v', '.sv', '.svh',
+            '.wat', '.wast', '.elm', '.purs', '.nix',
+            '.cmake', '.bzl', '.bazel', '.feature',
+            '.ps1', '.psm1', '.psd1', '.tcl', '.tk', '.awk',
+            '.mm', '.cls', '.trigger', '.abap', '.coffee', '.litcoffee', '.sas',
+            // Shaders & GPU
+            '.as', '.glsl', '.vert', '.frag', '.hlsl', '.fx', '.qs',
+            '.wgsl', '.metal', '.cu', '.cuh', '.fish',
+            '.scad', '.jsonnet', '.libsonnet',
+            '.re', '.rei', '.res', '.resi', '.sml', '.sig',
+            '.pony', '.fth', '.4th', '.chpl', '.x10', '.cecil', '.io', '.red', '.reds',
+            // Common config & data
+            '.sql', '.sh', '.bash', '.zsh',
+            '.json', '.yaml', '.yml', '.toml', '.xml',
+            '.html', '.css', '.scss', '.less',
+            '.vue', '.svelte', '.astro',
             '.md', '.mdx'
         ];
 
@@ -467,6 +1347,11 @@ class CFMemoryMCP {
                 } else if (entry.isFile()) {
                     const ext = path.extname(entry.name).toLowerCase();
 
+                    // Skip TypeScript declaration files — they're autogenerated
+                    // or framework types with thousands of identifiers that
+                    // aren't actual project code.
+                    if (entry.name.endsWith('.d.ts')) continue;
+
                     // Check if file should be excluded
                     const fileExcluded = effectiveExcludes.some(pattern => {
                         if (pattern.startsWith('*.')) {
@@ -520,8 +1405,9 @@ class CFMemoryMCP {
      */
     createAdaptiveBatches(files) {
         // Defaults tuned for Workers + typical project sizes.
-        // Max files per request keeps JSON parsing and request time stable.
-        const maxFiles = Number(process.env.CF_MEMORY_UPLOAD_BATCH_FILES || 100);
+        // Each file on the worker takes ~0.5-2s (hash + parse + embed + store).
+        // Keep batch size small enough that one batch fits comfortably in BATCH_TIMEOUT_MS.
+        const maxFiles = Number(process.env.CF_MEMORY_UPLOAD_BATCH_FILES || 25);
         // Max payload bytes (approx). Keep below a few MB to avoid timeouts.
         const maxBytes = Number(process.env.CF_MEMORY_UPLOAD_BATCH_BYTES || (1.5 * 1024 * 1024));
 
@@ -590,7 +1476,8 @@ class CFMemoryMCP {
                 path: batchPath,
                 method: 'POST',
                 headers,
-                timeout: TIMEOUT_MS
+                timeout: BATCH_TIMEOUT_MS,
+                agent: httpsAgent
             };
 
             const req = https.request(options, (res) => {
@@ -635,7 +1522,8 @@ class CFMemoryMCP {
                 path: cleanupPath,
                 method: 'POST',
                 headers,
-                timeout: TIMEOUT_MS
+                timeout: TIMEOUT_MS,
+                agent: httpsAgent
             };
 
             const req = https.request(options, (res) => {
@@ -657,6 +1545,114 @@ class CFMemoryMCP {
         });
     }
 
+    /**
+     * Compare each result's indexed_file_hash against the local file's
+     * current SHA-256 and set a `stale` field if they diverge. Works for
+     * the common case where the user has the project_path locally and
+     * the result file_paths are relative to it. Gracefully no-ops if we
+     * can't resolve the local path or read the file.
+     */
+    /**
+     * If retrieve_context is called without project_id and without
+     * all_projects, try to find a project whose root_path matches the
+     * current working directory (or CF_MEMORY_WATCH_PATH). Pass that
+     * project_id through. Cached for the process lifetime so we don't
+     * call list_projects on every retrieve.
+     */
+    async maybeFillProjectId(message) {
+        try {
+            const args = message.params && message.params.arguments;
+            if (!args || args.project_id || args.all_projects) return;
+            const cwd = process.env.CF_MEMORY_WATCH_PATH || process.cwd();
+            if (!cwd) return;
+
+            // Cache the resolution for the lifetime of this bridge process
+            // so we don't pay a list_projects round-trip per query.
+            if (!this._projectIdByCwdCache) this._projectIdByCwdCache = new Map();
+            const cached = this._projectIdByCwdCache.get(cwd);
+            if (cached !== undefined) {
+                if (cached) args.project_id = cached;
+                return;
+            }
+
+            const list = await this.makeRequest({
+                jsonrpc: '2.0',
+                id: `auto-proj-${Date.now()}`,
+                method: 'tools/call',
+                params: { name: 'list_projects', arguments: {} },
+            });
+            let projects = [];
+            try {
+                projects = JSON.parse(list.result.content[0].text);
+                if (!Array.isArray(projects)) projects = [];
+            } catch (_) {}
+
+            // Exact root_path match wins; otherwise prefix match (cwd is
+            // inside an indexed project).
+            const exact = projects.find(p => p.root_path === cwd);
+            const prefix = exact || projects.find(p =>
+                p.root_path && (cwd === p.root_path || cwd.startsWith(p.root_path + '/'))
+            );
+            const found = prefix?.id || null;
+            this._projectIdByCwdCache.set(cwd, found);
+            if (found) {
+                args.project_id = found;
+                _mcpTrace('AUTO_PROJECT', `cwd=${cwd} -> ${found}`);
+            }
+        } catch (err) {
+            this.logDebug(`maybeFillProjectId failed: ${err && err.message}`);
+        }
+    }
+
+    maybeAnnotateStaleness(response, args) {
+        try {
+            const text = response?.result?.content?.[0]?.text;
+            if (!text) return;
+            let parsed;
+            try { parsed = JSON.parse(text); } catch (_) { return; }
+            const results = Array.isArray(parsed?.results) ? parsed.results : null;
+            if (!results || results.length === 0) return;
+
+            // Try to find a local project root. We watch CF_MEMORY_WATCH_PATH
+            // (when auto-watch is on) or fall back to the cwd.
+            const root = process.env.CF_MEMORY_WATCH_PATH || process.cwd();
+            const stalePaths = new Set();
+            for (const r of results) {
+                if (!r || !r.file_path || !r.indexed_file_hash) continue;
+                const full = path.resolve(root, r.file_path);
+                let content;
+                try { content = fs.readFileSync(full, 'utf8'); } catch (_) { continue; }
+                // SHA-256 hex of UTF-8 content. Matches the server's hashContent().
+                const localHash = crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+                if (localHash !== r.indexed_file_hash) {
+                    r.stale = {
+                        last_indexed_at: r.indexed_at,
+                        reason: 'File edited locally since this chunk was indexed.',
+                    };
+                    stalePaths.add(r.file_path);
+                }
+            }
+            if (stalePaths.size > 0) {
+                parsed.stale_count = stalePaths.size;
+                // Concrete actionable hint with the actual file paths and the
+                // project_id from the first stale result. The model can copy
+                // this directly into a refresh_files call.
+                const projectId = results.find(r => r.stale && r.project_id)?.project_id;
+                parsed.stale_refresh_hint = {
+                    tool: 'refresh_files',
+                    arguments: {
+                        project_id: projectId,
+                        file_paths: Array.from(stalePaths),
+                    },
+                    alternative: 'Or call refresh_stale to refresh every stale file in the project at once.',
+                };
+                response.result.content[0].text = JSON.stringify(parsed);
+            }
+        } catch (err) {
+            this.logDebug(`maybeAnnotateStaleness failed: ${err && err.message}`);
+        }
+    }
+
     async makeRequest(message, extraHeaders = null) {
         return new Promise((resolve) => {
             const serverUrl = this.useStreamableHttp ? this.streamableHttpUrl : this.legacyServerUrl;
@@ -683,7 +1679,8 @@ class CFMemoryMCP {
                 path: url.pathname,
                 method: 'POST',
                 headers,
-                timeout: TIMEOUT_MS
+                timeout: TIMEOUT_MS,
+                agent: httpsAgent
             };
 
             const req = https.request(options, (res) => {
@@ -698,13 +1695,19 @@ class CFMemoryMCP {
                         const response = JSON.parse(body);
                         resolve(response);
                     } catch (error) {
+                        // Include HTTP status + body snippet so callers can
+                        // see whether they got a 404, a Cloudflare HTML error
+                        // page, or some other non-JSON response. Without this,
+                        // "Invalid JSON" alone hides whether the worker is
+                        // even reachable.
+                        const bodyPreview = body.slice(0, 200).replace(/\s+/g, ' ');
                         resolve({
                             jsonrpc: '2.0',
                             id: message.id || null,
                             error: {
                                 code: -32603,
-                                message: 'Invalid JSON response from server',
-                                data: error.message
+                                message: `Invalid JSON response from server (HTTP ${res.statusCode})`,
+                                data: `${error.message}; body[0..200]=${bodyPreview}`
                             }
                         });
                     }
@@ -805,9 +1808,11 @@ Usage:
   npx cf-memory-mcp                 Start the MCP server
   npx cf-memory-mcp --version       Show version
   npx cf-memory-mcp --help          Show this help
+  npx cf-memory-mcp --diagnose      Test connectivity and report issues
 
 Environment Variables:
   CF_MEMORY_API_KEY=<key>           Your CF Memory API key (required)
+  CF_MEMORY_BASE_URL=<url>          Override the default deployed worker
   CF_MEMORY_PROGRESS=true           Stream indexing progress to stderr (optional)
   DEBUG=1                           Enable debug logging
   MCP_DEBUG=1                       Enable MCP debug logging
@@ -817,6 +1822,149 @@ For more information, visit: https://github.com/johnlam90/cf-memory-mcp
     process.exit(0);
 }
 
+if (process.argv.includes('--diagnose')) {
+    (async () => {
+        console.log(`CF Memory MCP v${PACKAGE_VERSION} - Diagnostics`);
+        console.log(`Node.js: ${process.version}`);
+        console.log(`Platform: ${os.platform()} ${os.arch()}`);
+        console.log(`Target server: ${BASE_URL}`);
+        console.log(`API key set: ${API_KEY ? 'yes' : 'NO'}`);
+
+        if (!API_KEY) {
+            console.error('\nError: CF_MEMORY_API_KEY not set');
+            process.exit(1);
+        }
+
+        // Test 1: Health check
+        process.stdout.write('\n1. Health check... ');
+        const healthStart = Date.now();
+        try {
+            const url = new URL(BASE_URL + '/health');
+            const result = await new Promise((resolve, reject) => {
+                const req = https.request({
+                    hostname: url.hostname,
+                    port: url.port || 443,
+                    path: url.pathname,
+                    method: 'GET',
+                    timeout: 5000,
+                }, (res) => {
+                    let body = '';
+                    res.on('data', (c) => body += c);
+                    res.on('end', () => resolve({ status: res.statusCode, body }));
+                });
+                req.on('error', reject);
+                req.on('timeout', () => reject(new Error('timeout')));
+                req.end();
+            });
+            const elapsed = Date.now() - healthStart;
+            console.log(`OK (${elapsed}ms, HTTP ${result.status})`);
+        } catch (err) {
+            console.log(`FAIL: ${err.message}`);
+        }
+
+        // Test 2: MCP initialize
+        process.stdout.write('2. MCP initialize... ');
+        const initStart = Date.now();
+        try {
+            const result = await new Promise((resolve, reject) => {
+                const postData = JSON.stringify({
+                    jsonrpc: '2.0',
+                    id: 'diag',
+                    method: 'initialize',
+                    params: { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'diagnose', version: PACKAGE_VERSION } }
+                });
+                const url = new URL(STREAMABLE_HTTP_URL);
+                const req = https.request({
+                    hostname: url.hostname,
+                    port: url.port || 443,
+                    path: url.pathname,
+                    method: 'POST',
+                    timeout: 10000,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-API-Key': API_KEY,
+                        'Content-Length': Buffer.byteLength(postData)
+                    }
+                }, (res) => {
+                    let body = '';
+                    res.on('data', (c) => body += c);
+                    res.on('end', () => resolve({ status: res.statusCode, body }));
+                });
+                req.on('error', reject);
+                req.on('timeout', () => reject(new Error('timeout')));
+                req.write(postData);
+                req.end();
+            });
+            const elapsed = Date.now() - initStart;
+            const parsed = JSON.parse(result.body);
+            console.log(`OK (${elapsed}ms, ${parsed.result?.serverInfo?.name || 'unknown'})`);
+        } catch (err) {
+            console.log(`FAIL: ${err.message}`);
+        }
+
+        // Test 3: MCP tools/call list_projects
+        process.stdout.write('3. List projects... ');
+        const listStart = Date.now();
+        try {
+            const result = await new Promise((resolve, reject) => {
+                const postData = JSON.stringify({
+                    jsonrpc: '2.0',
+                    id: 'diag2',
+                    method: 'tools/call',
+                    params: { name: 'list_projects', arguments: {} }
+                });
+                const url = new URL(STREAMABLE_HTTP_URL);
+                const req = https.request({
+                    hostname: url.hostname,
+                    port: url.port || 443,
+                    path: url.pathname,
+                    method: 'POST',
+                    timeout: 10000,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-API-Key': API_KEY,
+                        'Content-Length': Buffer.byteLength(postData)
+                    }
+                }, (res) => {
+                    let body = '';
+                    res.on('data', (c) => body += c);
+                    res.on('end', () => resolve({ status: res.statusCode, body }));
+                });
+                req.on('error', reject);
+                req.on('timeout', () => reject(new Error('timeout')));
+                req.write(postData);
+                req.end();
+            });
+            const elapsed = Date.now() - listStart;
+            const parsed = JSON.parse(result.body);
+            const projects = JSON.parse(parsed.result?.content?.[0]?.text || '[]');
+            console.log(`OK (${elapsed}ms, ${projects.length} projects)`);
+
+            // 4. Auto-project detection for current cwd
+            process.stdout.write('4. Auto-project from cwd... ');
+            const cwd = process.env.CF_MEMORY_WATCH_PATH || process.cwd();
+            const exact = projects.find(p => p.root_path === cwd);
+            const prefix = exact || projects.find(p =>
+                p.root_path && (cwd === p.root_path || cwd.startsWith(p.root_path + '/'))
+            );
+            if (prefix) {
+                console.log(`OK (cwd=${cwd} -> ${prefix.name} [${prefix.id}])`);
+            } else {
+                console.log(`none (cwd=${cwd} doesn't match any project root)`);
+            }
+        } catch (err) {
+            console.log(`FAIL: ${err.message}`);
+        }
+
+        console.log('\nDiagnostics complete.');
+        process.exit(0);
+    })().catch(err => {
+        console.error('Diagnostic error:', err);
+        process.exit(1);
+    });
+    return;
+}
+
 // Check API key before starting server
 if (!API_KEY) {
     console.error('Error: CF_MEMORY_API_KEY environment variable is required');
@@ -827,7 +1975,7 @@ if (!API_KEY) {
     console.error('Or run with:');
     console.error('  CF_MEMORY_API_KEY="your-api-key-here" npx cf-memory-mcp');
     console.error('');
-    console.error('Get your API key from: https://cf-memory-mcp.johnlam90.workers.dev');
+    console.error(`Target server: ${BASE_URL}`);
     process.exit(1);
 }