@probelabs/probe 0.6.0-rc287 → 0.6.0-rc290

package/src/mcp/index.ts

@@ -21,11 +21,13 @@ import { fileURLToPath } from 'url';
 // Import from parent package
 import { search, query, extract, grep, getBinaryPath, setBinaryPath } from '../index.js';
 
+type OutputFormat = 'outline' | 'outline-xml' | 'json';
+
 // Parse command-line arguments
-function parseArgs(): { timeout?: number; format?: string } {
+function parseArgs(): { timeout?: number; lsp?: boolean; format?: OutputFormat } {
   const args = process.argv.slice(2);
-  const config: { timeout?: number; format?: string } = {};
-
+  const config: { timeout?: number; lsp?: boolean; format?: OutputFormat } = {};
+  
   for (let i = 0; i < args.length; i++) {
     if ((args[i] === '--timeout' || args[i] === '-t') && i + 1 < args.length) {
       const timeout = parseInt(args[i + 1], 10);
@@ -36,9 +38,17 @@ function parseArgs(): { timeout?: number; format?: string } {
         console.error(`Invalid timeout value: ${args[i + 1]}. Using default.`);
       }
       i++; // Skip the next argument
+    } else if (args[i] === '--lsp') {
+      config.lsp = true;
+      console.error('LSP mode enabled');
     } else if (args[i] === '--format' && i + 1 < args.length) {
-      config.format = args[i + 1];
-      console.error(`Format set to ${config.format}`);
+      const format = args[i + 1] as OutputFormat;
+      if (format === 'outline' || format === 'outline-xml' || format === 'json') {
+        config.format = format;
+        console.error(`Output format set to ${format}`);
+      } else {
+        console.error(`Invalid format value: ${args[i + 1]}. Using default.`);
+      }
       i++; // Skip the next argument
     } else if (args[i] === '--help' || args[i] === '-h') {
       console.error(`
@@ -49,7 +59,9 @@ Usage:
 
 Options:
   --timeout, -t <seconds>  Set timeout for search operations (default: 30)
-  --format <format>       Set output format (default: outline)
+  --lsp                    Enable LSP (Language Server Protocol) for enhanced features
+                           Automatically initializes language servers for the current workspace
+  --format <format>        Output format for search responses (outline|outline-xml|json)
   --help, -h              Show this help message
 `);
       process.exit(0);
@@ -120,17 +132,37 @@ interface SearchCodeArgs {
   exact?: boolean;
   strictElasticSyntax?: boolean;
   session?: string;
-  nextPage?: boolean;
+  timeout?: number;
+  noGitignore?: boolean;
+  lsp?: boolean;
+}
+
+interface QueryCodeArgs {
+  path: string;
+  pattern: string;
+  language?: string;
+  ignore?: string[];
+  allowTests?: boolean;
+  maxResults?: number;
+  format?: 'markdown' | 'plain' | 'json' | 'color';
+  timeout?: number;
+  noGitignore?: boolean;
 }
 
 interface ExtractCodeArgs {
   path: string;
   files: string[];
+  allowTests?: boolean;
+  contextLines?: number;
+  format?: 'markdown' | 'plain' | 'json';
+  timeout?: number;
+  noGitignore?: boolean;
+  lsp?: boolean;
 }
 
 interface GrepArgs {
   pattern: string;
-  paths: string | string[];
+  paths: string[];
   ignoreCase?: boolean;
   count?: boolean;
   context?: number;
@@ -139,11 +171,17 @@ interface GrepArgs {
 class ProbeServer {
   private server: Server;
   private defaultTimeout: number;
-  private defaultFormat?: string;
-
-  constructor(timeout: number = 30, format?: string) {
+  private lspEnabled: boolean;
+  private defaultFormat: OutputFormat;
+
+  constructor(
+    timeout: number = 30,
+    lspEnabled: boolean = false,
+    defaultFormat: OutputFormat = 'outline-xml'
+  ) {
     this.defaultTimeout = timeout;
-    this.defaultFormat = format;
+    this.lspEnabled = lspEnabled;
+    this.defaultFormat = defaultFormat;
     this.server = new Server(
       {
         name: '@probelabs/probe',
@@ -201,8 +239,11 @@ class ProbeServer {
               },
               nextPage: {
                 type: 'boolean',
-                description: 'Set to true when requesting the next page of results. Requires passing the same session ID from the previous search output.',
-                default: false
+                description: 'Skip .gitignore files (will use PROBE_NO_GITIGNORE environment variable if not set)',
+              },
+              lsp: {
+                type: 'boolean',
+                description: 'Use LSP (Language Server Protocol) for call hierarchy, reference counts, and enhanced symbol information',
               }
             },
             required: ['path', 'query']
@@ -221,7 +262,34 @@ class ProbeServer {
               files: {
                 type: 'array',
                 items: { type: 'string' },
-                description: 'Array of file paths to extract from. Formats: "file.js" (entire file), "file.js:42" (code block at line 42), "file.js:10-20" (lines 10-20), "file.js#funcName" (specific symbol). Line numbers and symbols are part of the path string, not separate parameters. Paths can be absolute or relative to the project directory.',
+                description: 'Files and lines or sybmbols to  extract from: /path/to/file.rs:10, /path/to/file.rs#func_name Path should be absolute.',
+              },
+              allowTests: {
+                type: 'boolean',
+                description: 'Allow test files and test code blocks in results (disabled by default)',
+              },
+              contextLines: {
+                type: 'number',
+                description: 'Number of context lines to include before and after the extracted block when AST parsing fails to find a suitable node',
+                default: 0
+              },
+              format: {
+                type: 'string',
+                enum: ['markdown', 'plain', 'json'],
+                description: 'Output format for the extracted code',
+                default: 'markdown'
+              },
+              timeout: {
+                type: 'number',
+                description: 'Timeout for the extract operation in seconds (default: 30)',
+              },
+              noGitignore: {
+                type: 'boolean',
+                description: 'Skip .gitignore files (will use PROBE_NO_GITIGNORE environment variable if not set)',
+              },
+              lsp: {
+                type: 'boolean',
+                description: 'Use LSP (Language Server Protocol) for call hierarchy, reference counts, and enhanced symbol information',
               }
             },
             required: ['path', 'files'],
@@ -365,7 +433,24 @@ class ProbeServer {
       } else if (this.defaultFormat === 'json') {
         options.json = true;
       }
-
+      if (args.session !== undefined && args.session.trim() !== '') {
+        options.session = args.session;
+      } else {
+        options.session = "new";
+      }
+      // Use timeout from args, or fall back to instance default
+      if (args.timeout !== undefined) {
+        options.timeout = args.timeout;
+      } else if (this.defaultTimeout !== undefined) {
+        options.timeout = this.defaultTimeout;
+      }
+      // Pass LSP flag if enabled globally or per-request
+      if (args.lsp !== undefined) {
+        options.lsp = args.lsp;
+      } else if (this.lspEnabled) {
+        options.lsp = true;
+      }
+      
       console.error("Executing search with options:", JSON.stringify(options, null, 2));
       
       try {
@@ -405,6 +490,19 @@ class ProbeServer {
         allowTests: true,  // Include test files by default
       };
       
+      // Use noGitignore from args, or fall back to PROBE_NO_GITIGNORE environment variable
+      if (args.noGitignore !== undefined) {
+        options.noGitignore = args.noGitignore;
+      } else if (process.env.PROBE_NO_GITIGNORE) {
+        options.noGitignore = process.env.PROBE_NO_GITIGNORE === 'true';
+      }
+      // Pass LSP flag if enabled globally or per-request
+      if (args.lsp !== undefined) {
+        options.lsp = args.lsp;
+      } else if (this.lspEnabled) {
+        options.lsp = true;
+      }
+      
       // Call extract with the complete options object
       try {
         // Track request size for token usage
@@ -507,6 +605,48 @@ class ProbeServer {
     // The @probelabs/probe package now handles binary path management internally
     // We don't need to verify or download the binary in the MCP server anymore
     
+    // Initialize LSP servers for the current workspace if --lsp flag is enabled
+    if (this.lspEnabled) {
+      const workspaceRoot = process.cwd();
+      console.error(`Initializing LSP servers for workspace: ${workspaceRoot}`);
+      
+      try {
+        // Execute probe lsp init command to pre-warm language servers
+        // Use recursive flag to discover nested projects in monorepos
+        const initCmd = process.platform === 'win32' 
+          ? `probe lsp init -w "${workspaceRoot}" --recursive`
+          : `probe lsp init -w '${workspaceRoot}' --recursive`;
+        
+        const { stdout, stderr } = await execAsync(initCmd, {
+          timeout: 10000, // 10 second timeout for initialization - don't wait too long
+          env: { ...process.env }
+        });
+        
+        if (stderr && !stderr.includes('Successfully initialized')) {
+          console.error(`LSP initialization warnings: ${stderr}`);
+        }
+        
+        console.error(`LSP servers initialized successfully for workspace: ${workspaceRoot}`);
+        
+        // Parse initialization output to show what was initialized
+        if (stdout) {
+          const lines = stdout.split('\n');
+          const initializedServers = lines.filter(line => 
+            line.includes('✓') || line.includes('language server')
+          );
+          if (initializedServers.length > 0) {
+            console.error('Initialized language servers:');
+            initializedServers.forEach(line => console.error(`  ${line.trim()}`));
+          }
+        }
+      } catch (error: any) {
+        // Don't fail MCP server startup if LSP initialization fails
+        // LSP will still work with cold start on first use
+        console.error(`Warning: Failed to initialize LSP servers: ${error.message || error}`);
+        console.error('LSP features will still be available but may have slower first-use performance');
+      }
+    }
+    
     // Just connect the server to the transport
     const transport = new StdioServerTransport();
     await this.server.connect(transport);
@@ -514,5 +654,10 @@ class ProbeServer {
   }
 }
 
-const server = new ProbeServer(cliConfig.timeout, cliConfig.format || 'outline');
+// Instantiate server with (timeout, lspEnabled, format)
+const server = new ProbeServer(
+  cliConfig.timeout ?? 30,
+  cliConfig.lsp ?? false,
+  cliConfig.format || 'outline-xml'
+);
 server.run().catch(console.error);

package/src/search.js

@@ -32,7 +32,8 @@ const SEARCH_FLAG_MAP = {
 	session: '--session',
 	timeout: '--timeout',
 	language: '--language',
-	format: '--format'
+	format: '--format',
+	lsp: '--lsp'
 };
 
 /**
@@ -58,7 +59,7 @@ const SEARCH_FLAG_MAP = {
  * @param {string} [options.session] - Session ID for caching results
  * @param {number} [options.timeout] - Timeout in seconds (default: 30)
  * @param {string} [options.language] - Limit search to files of a specific programming language
- * @param {string} [options.format] - Output format ('json', 'outline-xml', etc.)
+ * @param {boolean} [options.lsp] - Use LSP (Language Server Protocol) for enhanced symbol information
  * @param {Object} [options.binaryOptions] - Options for getting the binary
  * @param {boolean} [options.binaryOptions.forceDownload] - Force download even if binary exists
  * @param {string} [options.binaryOptions.version] - Specific version to download
@@ -85,8 +86,8 @@ export async function search(options) {
 	if (options.json && !options.format) {
 		cliArgs.push('--format', 'json');
 	} else if (options.format) {
-		// Format is already handled by buildCliArgs through SEARCH_FLAG_MAP
-		// but we need to ensure json parsing for json format
+		// Format is handled by buildCliArgs through SEARCH_FLAG_MAP.
+		// Ensure json parsing is enabled for json format.
 		if (options.format === 'json') {
 			options.json = true;
 		}
@@ -257,4 +258,4 @@ export async function search(options) {
 		};
 		throw structuredError;
 	}
-}
+}

package/src/tools/vercel.js

@@ -69,12 +69,12 @@ function autoQuoteSearchTerms(query) {
 		if (token.startsWith('"')) return token;
 		// Boolean operator
 		if (operators.has(token)) return token;
-		// Check if token needs quoting: has mixed case (upper+lower) or underscores
-		const hasUpper = /[A-Z]/.test(token);
-		const hasLower = /[a-z]/.test(token);
+		// Check if token needs quoting: has camelCase/PascalCase transitions or underscores
+		// Simple capitalized words like "Redis" or "Limiter" should NOT be quoted —
+		// only quote when there's an actual case transition (e.g., "getUserData", "NewSlidingLog")
 		const hasUnderscore = token.includes('_');
-		const hasMixedCase = hasUpper && hasLower;
-		if (hasMixedCase || hasUnderscore) {
+		const hasCaseTransition = /[a-z][A-Z]/.test(token) || /[A-Z]{2,}[a-z]/.test(token);
+		if (hasCaseTransition || hasUnderscore) {
 			return `"${token}"`;
 		}
 		return token;
@@ -237,7 +237,7 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'Break down complex queries into multiple searches to cover all aspects.',
 		'',
 		'Available tools:',
-		'- search: Find code matching keywords or patterns. Run multiple searches for different aspects of complex queries.',
+		'- search: Find code matching keywords or patterns. Results are paginated — use nextPage=true when results are relevant to get more. Run multiple searches for different aspects.',
 		'- extract: Verify code snippets to ensure targets are actually relevant before including them.',
 		'- listFiles: Understand directory structure to find where relevant code might live.',
 		'',
@@ -258,13 +258,14 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'',
 		'Combining searches with OR:',
 		'- Multiple unquoted words use OR logic: rate limit matches files containing EITHER "rate" OR "limit".',
-		'- For known symbol names, quote each term to prevent splitting: \'"limitDRL" "limitRedis"\' matches either exact symbol.',
+		'- IMPORTANT: Multiple quoted terms use AND logic by default: \'"RateLimit" "middleware"\' requires BOTH in the same file.',
+		'- To search for ANY of several quoted symbols, use the explicit OR operator: \'"ForwardMessage" OR "SessionLimiter"\'.',
 		'- Without quotes, camelCase like limitDRL gets split into "limit" + "DRL" — not what you want for symbol lookup.',
 		'- Use OR to search for multiple related symbols in ONE search instead of separate searches.',
 		'- This is much faster than running separate searches sequentially.',
-		'- Example: search \'"ForwardMessage" "SessionLimiter"\' finds files with either exact symbol in one call.',
-		'- Example: search \'"limitDRL" "doRollingWindowWrite"\' finds both rate limiting functions at once.',
-		'- Use AND only when you need both terms to appear in the same file: "rate AND limit".',
+		'- Example: search \'"ForwardMessage" OR "SessionLimiter"\' finds files with either exact symbol in one call.',
+		'- Example: search \'"limitDRL" OR "doRollingWindowWrite"\' finds both rate limiting functions at once.',
+		'- Use AND (or just put quoted terms together) when you need both terms in the same file.',
 		'',
 		'Parallel tool calls:',
 		'- When you need to search for INDEPENDENT concepts, call multiple search tools IN PARALLEL (same response).',
@@ -278,10 +279,10 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'  Query: "Find the IP allowlist middleware"',
 		'  → search "allowlist middleware" (one search, probe handles IP/ip/Ip variations)',
 		'  Query: "Find ForwardMessage and SessionLimiter"',
-		'  → search \'"ForwardMessage" "SessionLimiter"\' (one OR search finds both exact symbols)',
+		'  → search \'"ForwardMessage" OR "SessionLimiter"\' (one OR search finds both exact symbols)',
 		'  OR: search exact=true "ForwardMessage" + search exact=true "SessionLimiter" IN PARALLEL',
 		'  Query: "Find limitDRL and limitRedis functions"',
-		'  → search \'"limitDRL" "limitRedis"\' (one OR search, quoted to prevent camelCase splitting)',
+		'  → search \'"limitDRL" OR "limitRedis"\' (one OR search, quoted to prevent camelCase splitting)',
 		'  Query: "Find ThrottleRetryLimit usage"',
 		'  → search exact=true "ThrottleRetryLimit" (one search, if no results the symbol does not exist — stop)',
 		'  Query: "How does BM25 scoring work with SIMD optimization?"',
@@ -289,7 +290,7 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'',
 		'BAD search strategy (never do this):',
 		'  → search "AllowedIPs" → search "allowedIps" → search "allowed_ips" (WRONG: case/style variations, probe handles them)',
-		'  → search "limitDRL" → search "LimitDRL" (WRONG: case variation — combine with OR: \'"limitDRL" "limitRedis"\')',
+		'  → search "limitDRL" → search "LimitDRL" (WRONG: case variation — combine with OR: \'"limitDRL" OR "limitRedis"\')',
 		'  → search "throttle_retry_limit" after searching "ThrottleRetryLimit" (WRONG: snake_case variation, probe handles it)',
 		'  → search "ThrottleRetryLimit" path=tyk → search "ThrottleRetryLimit" path=gateway → search "ThrottleRetryLimit" path=apidef (WRONG: same query on different paths — probe searches recursively)',
 		'  → search "func (k *RateLimitAndQuotaCheck) handleRateLimitFailure" (WRONG: do not search full function signatures, just use exact=true "handleRateLimitFailure")',
@@ -302,15 +303,34 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'- To bypass stopword filtering: wrap terms in quotes ("return", "struct") or set exact=true. Both disable stemming and splitting too.',
 		'- camelCase terms are split: getUserData becomes "get", "user", "data" — so one search covers all naming styles.',
 		'- Do NOT search for full function signatures like "func (r *Type) Method(args)". Just search for the method name with exact=true.',
+		'- Do NOT search for file names (e.g., "sliding_log.go"). Use listFiles to discover files by name.',
+		'',
+		'PAGINATION:',
+		'- Search results are paginated (~20k tokens per page).',
+		'- If your search returned relevant files, call the same query with nextPage=true to check for more.',
+		'- Keep paginating while results stay relevant. Stop when results are off-topic or "All results retrieved".',
+		'',
+		'WHEN TO STOP:',
+		'- After you have explored the main concept AND related subsystems.',
+		'- Once you have 5-15 targets covering different aspects of the query.',
+		'- If you get a "DUPLICATE SEARCH BLOCKED" message, move on.',
 		'',
 		'Strategy:',
-		'1. Analyze the query - identify key concepts and group related symbols',
-		'2. Combine related symbols into OR searches: \'"symbolA" "symbolB"\' finds files with either (quote to prevent splitting)',
-		'3. Run INDEPENDENT searches in PARALLEL — do not wait for one to finish before starting another',
+		'1. Analyze the query — identify key concepts, then brainstorm SYNONYMS and alternative terms for each.',
+		'   Code naming often differs from the concept: "authentication" → verify, credentials, login, auth;',
+		'   "rate limiting" → throttle, quota, limiter, bucket; "error handling" → catch, recover, panic.',
+		'   Think about what a developer would NAME the function/struct/variable, not just the concept.',
+		'2. Run INDEPENDENT searches in PARALLEL — search for the main concept AND synonyms simultaneously.',
+		'   After each search, check if results are relevant. If yes, call nextPage=true for more results.',
+		'3. Combine related symbols into OR searches: \'"symbolA" OR "symbolB"\' finds files with either.',
 		'4. For known symbol names use exact=true. For concepts use default (exact=false).',
-		'5. If a search returns results, use extract to verify relevance. Run multiple extracts in parallel too.',
-		'6. If a search returns NO results, the term does not exist. Do NOT retry with variations, different paths, or longer strings. Move on.',
-		'7. Combine all relevant targets in your final response',
+		'5. After your first round of searches, READ the extracted code and look for connected code:',
+		'   - Function calls to other important functions → include those targets.',
+		'   - Type references and imports → include type definitions.',
+		'   - Registered handlers/middleware → include all registered items.',
+		'6. If a search returns results, use extract to verify relevance. Run multiple extracts in parallel too.',
+		'7. If a search returns NO results, the term does not exist. Do NOT retry with variations. Move on.',
+		'8. Once you have enough targets (typically 5-15), output your final JSON answer immediately.',
 		'',
 		`Query: ${searchQuery}`,
 		`Search path(s): ${searchPath}`,
@@ -319,7 +339,9 @@ function buildSearchDelegateTask({ searchQuery, searchPath, exact, language, all
 		'Return ONLY valid JSON: {"targets": ["path/to/file.ext#Symbol", "path/to/file.ext:line", "path/to/file.ext:start-end"]}',
 		'IMPORTANT: Use ABSOLUTE file paths in targets (e.g., "/full/path/to/file.ext#Symbol"). If you only have relative paths, make them relative to the search path above.',
 		'Prefer #Symbol when a function/class name is clear; otherwise use line numbers.',
-		'Deduplicate targets. Do NOT explain or answer - ONLY return the JSON targets.'
+		'Deduplicate targets. Do NOT explain or answer - ONLY return the JSON targets.',
+		'',
+		'Remember: if your search returned relevant results, use nextPage=true to check for more before outputting.'
 	].join('\n');
 }
 
@@ -351,6 +373,8 @@ export const searchTool = (options = {}) => {
 
 	// Track previous non-paginated searches to detect and block duplicates
 	const previousSearches = new Set();
+	// Track how many times a duplicate search has been blocked (for escalating messages)
+	let consecutiveDupBlocks = 0;
 	// Track pagination counts per query to cap runaway pagination
 	const paginationCounts = new Map();
 	const MAX_PAGES_PER_QUERY = 3;
@@ -422,12 +446,17 @@ export const searchTool = (options = {}) => {
 				const searchKey = `${searchQuery}::${exact || false}`;
 				if (!nextPage) {
 					if (previousSearches.has(searchKey)) {
+						consecutiveDupBlocks++;
 						if (debug) {
-							console.error(`[DEDUP] Blocked duplicate search: "${searchQuery}" (path: "${searchPath}")`);
+							console.error(`[DEDUP] Blocked duplicate search (${consecutiveDupBlocks}x): "${searchQuery}" (path: "${searchPath}")`);
+						}
+						if (consecutiveDupBlocks >= 3) {
+							return 'STOP. You have been blocked ' + consecutiveDupBlocks + ' times for repeating searches. You MUST output your final JSON answer NOW with whatever targets you have found. Do NOT call any more tools.';
 						}
-						return 'DUPLICATE SEARCH BLOCKED: You already searched for this exact query. Changing the path does NOT give different results — probe searches recursively. Do NOT repeat the same search. Try a genuinely different keyword, use extract to examine results you already found, or provide your final answer if you have enough information.';
+						return 'DUPLICATE SEARCH BLOCKED (' + consecutiveDupBlocks + 'x). You already searched for this. Do NOT repeat — probe searches recursively across all paths. Either: (1) use extract on results you already found, (2) try a COMPLETELY different keyword, or (3) output your final answer NOW.';
 					}
 					previousSearches.add(searchKey);
+					consecutiveDupBlocks = 0; // Reset on successful new search
 					paginationCounts.set(searchKey, 0);
 				} else {
 					// Cap pagination to prevent runaway page-through of broad queries