@probelabs/probe 0.6.0-rc116 → 0.6.0-rc118

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@probelabs/probe",
-  "version": "0.6.0-rc116",
+  "version": "0.6.0-rc118",
   "description": "Node.js wrapper for the probe code search tool",
   "main": "src/index.js",
   "module": "src/index.js",
@@ -85,8 +85,8 @@
     "axios": "^1.8.3",
     "fs-extra": "^11.1.1",
     "glob": "^10.3.10",
+    "gpt-tokenizer": "^3.0.1",
     "tar": "^6.2.0",
-    "tiktoken": "^1.0.20",
     "zod": "^3.24.2"
   },
   "devDependencies": {

package/src/agent/ProbeAgent.js

@@ -188,9 +188,23 @@ export class ProbeAgent {
     if (this.enableBash && wrappedTools.bashToolInstance) {
       this.toolImplementations.bash = wrappedTools.bashToolInstance;
     }
-    
+
     // Store wrapped tools for ACP system
     this.wrappedTools = wrappedTools;
+
+    // Log available tools in debug mode
+    if (this.debug) {
+      console.error('\n[DEBUG] ========================================');
+      console.error('[DEBUG] ProbeAgent Tools Initialized');
+      console.error('[DEBUG] Session ID:', this.sessionId);
+      console.error('[DEBUG] Available tools:');
+      for (const toolName of Object.keys(this.toolImplementations)) {
+        console.error(`[DEBUG]   - ${toolName}`);
+      }
+      console.error('[DEBUG] Allowed folders:', this.allowedFolders);
+      console.error('[DEBUG] Outline mode:', this.outline);
+      console.error('[DEBUG] ========================================\n');
+    }
   }
 
   /**
@@ -366,6 +380,9 @@ export class ProbeAgent {
   async processImageReferences(content) {
     if (!content) return;
 
+    // First, try to parse listFiles output format to extract directory context
+    const listFilesDirectories = this.extractListFilesDirectories(content);
+
     // Enhanced pattern to detect image file mentions in various contexts
     // Looks for: "image", "file", "screenshot", etc. followed by path-like strings with image extensions
     const extensionsPattern = `(?:${SUPPORTED_IMAGE_EXTENSIONS.join('|')})`;
@@ -400,10 +417,55 @@ export class ProbeAgent {
 
     // Process each found path
     for (const imagePath of foundPaths) {
-      await this.loadImageIfValid(imagePath);
+      // Try to resolve the path with directory context from listFiles output
+      let resolvedPath = imagePath;
+
+      // If the path is just a filename (no directory separator), try to find it in listFiles directories
+      if (!imagePath.includes('/') && !imagePath.includes('\\')) {
+        for (const dir of listFilesDirectories) {
+          const potentialPath = resolve(dir, imagePath);
+          // Check if this file exists by attempting to load it
+          const loaded = await this.loadImageIfValid(potentialPath);
+          if (loaded) {
+            // Successfully loaded with this directory context
+            if (this.debug) {
+              console.log(`[DEBUG] Resolved ${imagePath} to ${potentialPath} using listFiles context`);
+            }
+            break; // Found it, no need to try other directories
+          }
+        }
+      } else {
+        // Path already has directory info, load as-is
+        await this.loadImageIfValid(resolvedPath);
+      }
     }
   }
 
+  /**
+   * Extract directory paths from listFiles tool output
+   * @param {string} content - Tool output content
+   * @returns {string[]} - Array of directory paths
+   */
+  extractListFilesDirectories(content) {
+    const directories = [];
+
+    // Pattern to match listFiles output format: "/path/to/directory:" at the start of a line
+    const dirPattern = /^([^\n:]+):\s*$/gm;
+
+    let match;
+    while ((match = dirPattern.exec(content)) !== null) {
+      const dirPath = match[1].trim();
+      if (dirPath && dirPath.length > 0) {
+        directories.push(dirPath);
+        if (this.debug) {
+          console.log(`[DEBUG] Extracted directory context from listFiles: ${dirPath}`);
+        }
+      }
+    }
+
+    return directories;
+  }
+
   /**
    * Load and cache an image if it's valid and accessible
    * @param {string} imagePath - Path to the image file
@@ -630,16 +692,23 @@ export class ProbeAgent {
       this.mcpBridge = new MCPXmlBridge({ debug: this.debug });
       await this.mcpBridge.initialize(mcpConfig);
 
-      const mcpToolCount = this.mcpBridge.getToolNames().length;
+      const mcpToolNames = this.mcpBridge.getToolNames();
+      const mcpToolCount = mcpToolNames.length;
       if (mcpToolCount > 0) {
         if (this.debug) {
-          console.log(`[DEBUG] Loaded ${mcpToolCount} MCP tools`);
+          console.error('\n[DEBUG] ========================================');
+          console.error(`[DEBUG] MCP Tools Initialized (${mcpToolCount} tools)`);
+          console.error('[DEBUG] Available MCP tools:');
+          for (const toolName of mcpToolNames) {
+            console.error(`[DEBUG]   - ${toolName}`);
+          }
+          console.error('[DEBUG] ========================================\n');
         }
       } else {
         // For backward compatibility: if no tools were loaded, set bridge to null
         // This maintains the behavior expected by existing tests
         if (this.debug) {
-          console.log('[DEBUG] No MCP tools loaded, setting bridge to null');
+          console.error('[DEBUG] No MCP tools loaded, setting bridge to null');
         }
         this.mcpBridge = null;
       }
@@ -1117,34 +1186,74 @@ When troubleshooting:
             if (type === 'mcp' && this.mcpBridge && this.mcpBridge.isMcpTool(toolName)) {
               // Execute MCP tool
               try {
-                if (this.debug) console.log(`[DEBUG] Executing MCP tool '${toolName}' with params:`, params);
+                // Log MCP tool execution in debug mode
+                if (this.debug) {
+                  console.error(`\n[DEBUG] ========================================`);
+                  console.error(`[DEBUG] Executing MCP tool: ${toolName}`);
+                  console.error(`[DEBUG] Arguments:`);
+                  for (const [key, value] of Object.entries(params)) {
+                    const displayValue = typeof value === 'string' && value.length > 100
+                      ? value.substring(0, 100) + '...'
+                      : value;
+                    console.error(`[DEBUG]   ${key}: ${JSON.stringify(displayValue)}`);
+                  }
+                  console.error(`[DEBUG] ========================================\n`);
+                }
 
                 // Execute MCP tool through the bridge
                 const executionResult = await this.mcpBridge.mcpTools[toolName].execute(params);
 
                 const toolResultContent = typeof executionResult === 'string' ? executionResult : JSON.stringify(executionResult, null, 2);
-                const preview = createMessagePreview(toolResultContent);
+
+                // Log MCP tool result in debug mode
                 if (this.debug) {
-                  console.log(`[DEBUG] MCP tool '${toolName}' executed successfully. Result preview: ${preview}`);
+                  const preview = toolResultContent.length > 500 ? toolResultContent.substring(0, 500) + '...' : toolResultContent;
+                  console.error(`[DEBUG] ========================================`);
+                  console.error(`[DEBUG] MCP tool '${toolName}' completed successfully`);
+                  console.error(`[DEBUG] Result preview:`);
+                  console.error(preview);
+                  console.error(`[DEBUG] ========================================\n`);
                 }
 
                 currentMessages.push({ role: 'user', content: `<tool_result>\n${toolResultContent}\n</tool_result>` });
               } catch (error) {
                 console.error(`Error executing MCP tool ${toolName}:`, error);
                 const toolResultContent = `Error executing MCP tool ${toolName}: ${error.message}`;
-                if (this.debug) console.log(`[DEBUG] MCP tool '${toolName}' execution FAILED.`);
+
+                // Log MCP tool error in debug mode
+                if (this.debug) {
+                  console.error(`[DEBUG] ========================================`);
+                  console.error(`[DEBUG] MCP tool '${toolName}' failed with error:`);
+                  console.error(`[DEBUG] ${error.message}`);
+                  console.error(`[DEBUG] ========================================\n`);
+                }
+
                 currentMessages.push({ role: 'user', content: `<tool_result>\n${toolResultContent}\n</tool_result>` });
               }
             } else if (this.toolImplementations[toolName]) {
               // Execute native tool
               try {
                 // Add sessionId and workingDirectory to params for tool execution
-                const toolParams = { 
-                  ...params, 
+                const toolParams = {
+                  ...params,
                   sessionId: this.sessionId,
                   workingDirectory: (this.allowedFolders && this.allowedFolders[0]) || process.cwd()
                 };
-                
+
+                // Log tool execution in debug mode
+                if (this.debug) {
+                  console.error(`\n[DEBUG] ========================================`);
+                  console.error(`[DEBUG] Executing tool: ${toolName}`);
+                  console.error(`[DEBUG] Arguments:`);
+                  for (const [key, value] of Object.entries(params)) {
+                    const displayValue = typeof value === 'string' && value.length > 100
+                      ? value.substring(0, 100) + '...'
+                      : value;
+                    console.error(`[DEBUG]   ${key}: ${JSON.stringify(displayValue)}`);
+                  }
+                  console.error(`[DEBUG] ========================================\n`);
+                }
+
                 // Emit tool start event
                 this.events.emit('toolCall', {
                   timestamp: new Date().toISOString(),
@@ -1196,6 +1305,18 @@ When troubleshooting:
                     toolResult = await executeToolCall();
                   }
                   
+                  // Log tool result in debug mode
+                  if (this.debug) {
+                    const resultPreview = typeof toolResult === 'string'
+                      ? (toolResult.length > 500 ? toolResult.substring(0, 500) + '...' : toolResult)
+                      : (toolResult ? JSON.stringify(toolResult, null, 2).substring(0, 500) + '...' : 'No Result');
+                    console.error(`[DEBUG] ========================================`);
+                    console.error(`[DEBUG] Tool '${toolName}' completed successfully`);
+                    console.error(`[DEBUG] Result preview:`);
+                    console.error(resultPreview);
+                    console.error(`[DEBUG] ========================================\n`);
+                  }
+
                   // Emit tool success event
                   this.events.emit('toolCall', {
                     timestamp: new Date().toISOString(),
@@ -1206,8 +1327,16 @@ When troubleshooting:
                       : (toolResult ? JSON.stringify(toolResult).substring(0, 200) + '...' : 'No Result'),
                     status: 'completed'
                   });
-                  
+
                 } catch (toolError) {
+                  // Log tool error in debug mode
+                  if (this.debug) {
+                    console.error(`[DEBUG] ========================================`);
+                    console.error(`[DEBUG] Tool '${toolName}' failed with error:`);
+                    console.error(`[DEBUG] ${toolError.message}`);
+                    console.error(`[DEBUG] ========================================\n`);
+                  }
+
                   // Emit tool error event
                   this.events.emit('toolCall', {
                     timestamp: new Date().toISOString(),
@@ -1262,7 +1391,23 @@ When troubleshooting:
             }
           }
         } else {
-          // No tool call found, add assistant response and ask for tool usage
+          // No tool call found
+          // Special case: If response contains a mermaid code block and no schema was provided,
+          // treat it as a valid completion (for mermaid diagram fixing workflow)
+          const hasMermaidCodeBlock = /```mermaid\s*\n[\s\S]*?\n```/.test(assistantResponseContent);
+          const hasNoSchemaOrTools = !options.schema && validTools.length === 0;
+
+          if (hasMermaidCodeBlock && hasNoSchemaOrTools) {
+            // Accept mermaid code block as final answer for diagram fixing
+            finalResult = assistantResponseContent;
+            completionAttempted = true;
+            if (this.debug) {
+              console.error(`[DEBUG] Accepting mermaid code block as valid completion (no schema, no tools)`);
+            }
+            break;
+          }
+
+          // Add assistant response and ask for tool usage
           currentMessages.push({ role: 'assistant', content: assistantResponseContent });
 
           // Build appropriate reminder message based on whether schema is provided

package/src/agent/probeTool.js

@@ -213,65 +213,166 @@ export function createWrappedTools(baseTools) {
   return wrappedTools;
 }
 
-// Simple file listing tool
+// Simple file listing tool with ls-like formatted output
 export const listFilesTool = {
   execute: async (params) => {
     const { directory = '.', workingDirectory } = params;
-    
+
     // Use the provided working directory, or fall back to process.cwd()
     const baseCwd = workingDirectory || process.cwd();
-    
+
     // Security: Validate path to prevent traversal attacks
     const secureBaseDir = path.resolve(baseCwd);
-    const targetDir = path.resolve(secureBaseDir, directory);
-    if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
-      throw new Error('Path traversal attempt detected. Access denied.');
+
+    // If directory is absolute, check if it's within the secure base directory
+    // If it's relative, resolve it against the secure base directory
+    let targetDir;
+    if (path.isAbsolute(directory)) {
+      targetDir = path.resolve(directory);
+      // Check if the absolute path is within the secure base directory
+      if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
+        throw new Error(`Path traversal attempt detected. Cannot access directory outside workspace: ${directory}`);
+      }
+    } else {
+      targetDir = path.resolve(secureBaseDir, directory);
+      // Double-check the resolved path is still within the secure base directory
+      if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
+        throw new Error(`Path traversal attempt detected. Access denied: ${directory}`);
+      }
     }
-    
+
+    const debug = process.env.DEBUG === '1';
+
+    if (debug) {
+      console.log(`[DEBUG] Listing files in directory: ${targetDir}`);
+    }
+
     try {
-      const files = await listFilesByLevel({
-        directory: targetDir,
-        maxFiles: 100,
-        respectGitignore: !process.env.PROBE_NO_GITIGNORE || process.env.PROBE_NO_GITIGNORE === '',
-        cwd: secureBaseDir
+      // Read the directory contents
+      const files = await fsPromises.readdir(targetDir, { withFileTypes: true });
+
+      // Format size for human readability
+      const formatSize = (size) => {
+        if (size < 1024) return `${size}B`;
+        if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)}K`;
+        if (size < 1024 * 1024 * 1024) return `${(size / (1024 * 1024)).toFixed(1)}M`;
+        return `${(size / (1024 * 1024 * 1024)).toFixed(1)}G`;
+      };
+
+      // Format the results as ls-style output
+      const entries = await Promise.all(files.map(async (file) => {
+        const isDirectory = file.isDirectory();
+        const fullPath = path.join(targetDir, file.name);
+
+        let size = 0;
+        try {
+          const stats = await fsPromises.stat(fullPath);
+          size = stats.size;
+        } catch (statError) {
+          if (debug) {
+            console.log(`[DEBUG] Could not stat file ${file.name}:`, statError.message);
+          }
+        }
+
+        return {
+          name: file.name,
+          isDirectory,
+          size
+        };
+      }));
+
+      // Sort: directories first, then files, both alphabetically
+      entries.sort((a, b) => {
+        if (a.isDirectory && !b.isDirectory) return -1;
+        if (!a.isDirectory && b.isDirectory) return 1;
+        return a.name.localeCompare(b.name);
       });
-      
-      return files;
+
+      // Format entries
+      const formatted = entries.map(entry => {
+        const type = entry.isDirectory ? 'dir ' : 'file';
+        const sizeStr = formatSize(entry.size).padStart(8);
+        return `${type} ${sizeStr}  ${entry.name}`;
+      });
+
+      if (debug) {
+        console.log(`[DEBUG] Found ${entries.length} files/directories in ${targetDir}`);
+      }
+
+      // Return as formatted text output
+      const header = `${targetDir}:\n`;
+      const output = header + formatted.join('\n');
+
+      return output;
     } catch (error) {
       throw new Error(`Failed to list files: ${error.message}`);
     }
   }
 };
 
-// Simple file search tool
+// Simple file search tool with timeout protection
 export const searchFilesTool = {
   execute: async (params) => {
     const { pattern, directory = '.', recursive = true, workingDirectory } = params;
-    
+
     if (!pattern) {
       throw new Error('Pattern is required for file search');
     }
-    
+
     // Security: Validate path to prevent traversal attacks
     const baseCwd = workingDirectory || process.cwd();
     const secureBaseDir = path.resolve(baseCwd);
-    const targetDir = path.resolve(secureBaseDir, directory);
-    if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
-      throw new Error('Path traversal attempt detected. Access denied.');
+
+    // If directory is absolute, check if it's within the secure base directory
+    // If it's relative, resolve it against the secure base directory
+    let targetDir;
+    if (path.isAbsolute(directory)) {
+      targetDir = path.resolve(directory);
+      // Check if the absolute path is within the secure base directory
+      if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
+        throw new Error(`Path traversal attempt detected. Cannot access directory outside workspace: ${directory}`);
+      }
+    } else {
+      targetDir = path.resolve(secureBaseDir, directory);
+      // Double-check the resolved path is still within the secure base directory
+      if (!targetDir.startsWith(secureBaseDir + path.sep) && targetDir !== secureBaseDir) {
+        throw new Error(`Path traversal attempt detected. Access denied: ${directory}`);
+      }
     }
-    
+
+    // Validate pattern complexity to prevent DoS
+    if (pattern.includes('**/**') || pattern.split('*').length > 10) {
+      throw new Error('Pattern too complex. Please use a simpler glob pattern.');
+    }
+
     try {
       const options = {
         cwd: targetDir,
         ignore: ['node_modules/**', '.git/**'],
         absolute: false
       };
-      
+
       if (!recursive) {
         options.deep = 1;
       }
-      
-      const files = await glob(pattern, options);
+
+      // Create a timeout promise (10 seconds)
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => reject(new Error('Search operation timed out after 10 seconds')), 10000);
+      });
+
+      // Race glob against timeout
+      const files = await Promise.race([
+        glob(pattern, options),
+        timeoutPromise
+      ]);
+
+      // Limit results to prevent memory issues
+      const maxResults = 1000;
+      if (files.length > maxResults) {
+        return files.slice(0, maxResults);
+      }
+
       return files;
     } catch (error) {
       throw new Error(`Failed to search files: ${error.message}`);

package/src/agent/schemaUtils.js

@@ -813,7 +813,7 @@ When presented with a broken Mermaid diagram, analyze it thoroughly and provide
         debug: this.options.debug,
         tracer: this.options.tracer,
         allowEdit: this.options.allowEdit,
-        maxIterations: 2,  // Limit mermaid fixing to 2 iterations to prevent long loops
+        maxIterations: 10,  // Allow more iterations for mermaid fixing to handle complex diagrams
         disableMermaidValidation: true  // CRITICAL: Disable mermaid validation in nested agent to prevent infinite recursion
       });
     }

package/src/agent/tokenCounter.js

@@ -1,14 +1,14 @@
-import { get_encoding } from 'tiktoken';
+import { encode } from 'gpt-tokenizer';
 
 /**
  * TokenCounter class to track token usage in the agent
  */
 export class TokenCounter {
   constructor() {
-    // Initialize the tokenizer with cl100k_base encoding (works for both Claude and GPT models)
+    // Initialize the tokenizer with gpt-tokenizer (works for both Claude and GPT models)
     try {
-      // Initialize tokenizer
-      this.tokenizer = get_encoding('cl100k_base');
+      // gpt-tokenizer uses encode directly, no need to initialize
+      this.tokenizer = encode;
 
       // Context window tracking
       this.contextSize = 0; // Current size based on history
@@ -30,7 +30,7 @@ export class TokenCounter {
 
     } catch (error) {
       console.error('Error initializing tokenizer:', error);
-      // Fallback to a simple token counting method if tiktoken fails
+      // Fallback to a simple token counting method if gpt-tokenizer fails
       this.tokenizer = null;
       this.contextSize = 0;
       this.requestTokens = 0;
@@ -49,7 +49,7 @@ export class TokenCounter {
   }
 
   /**
-   * Count tokens in a string using tiktoken or fallback method
+   * Count tokens in a string using gpt-tokenizer or fallback method
    * @param {string} text - The text to count tokens for
    * @returns {number} - The number of tokens
    */
@@ -60,7 +60,7 @@ export class TokenCounter {
 
     if (this.tokenizer) {
       try {
-        const tokens = this.tokenizer.encode(text);
+        const tokens = this.tokenizer(text);
         return tokens.length;
       } catch (error) {
         // Fallback to a simple approximation (1 token ≈ 4 characters)

package/src/index.js

@@ -34,6 +34,7 @@ import { searchTool, queryTool, extractTool, delegateTool } from './tools/vercel
 import { bashTool } from './tools/bash.js';
 import { ProbeAgent } from './agent/ProbeAgent.js';
 import { SimpleTelemetry, SimpleAppTracer, initializeSimpleTelemetryFromOptions } from './agent/simpleTelemetry.js';
+import { listFilesToolInstance, searchFilesToolInstance } from './agent/probeTool.js';
 
 export {
 	search,
@@ -57,6 +58,9 @@ export {
 	extractTool,
 	delegateTool,
 	bashTool,
+	// Export tool instances
+	listFilesToolInstance,
+	searchFilesToolInstance,
 	// Export schemas
 	searchSchema,
 	querySchema,