onbuzz 3.3.0

package/src/tools/browserTool.js

@@ -0,0 +1,897 @@
+/**
+ * @deprecated This tool is DEPRECATED as of December 2024.
+ * Use WebTool (toolId: "web") instead for all browser automation tasks.
+ *
+ * WebTool provides:
+ * - All browser automation features (navigate, click, type, screenshot, etc.)
+ * - Built-in search engine support (Google, Bing, DuckDuckGo)
+ * - Tab management with agent isolation
+ * - AI-powered screenshot analysis
+ *
+ * This file is kept for backward compatibility but will be removed in a future version.
+ *
+ * Migration guide:
+ * - Replace toolId "browser" with "web"
+ * - Use operation: "interactive" with actions array
+ * - Wrap actions in open-tab for tab context
+ *
+ * Original BrowserTool - Handle browser automation and web interaction tasks
+ *
+ * Purpose:
+ * - Automate web browser interactions
+ * - Navigate websites and extract data
+ * - Fill forms and interact with web elements
+ * - Take screenshots and analyze web content
+ * - Handle authentication and sessions
+ */
+
+import { BaseTool } from './baseTool.js';
+import TagParser from '../utilities/tagParser.js';
+import puppeteer from 'puppeteer';
+import fs from 'fs/promises';
+import path from 'path';
+
+import {
+  TOOL_STATUS,
+  SYSTEM_DEFAULTS
+} from '../utilities/constants.js';
+import { getDefaultUserAgent } from '../utilities/platformUtils.js';
+
+class BrowserTool extends BaseTool {
+  constructor(config = {}, logger = null) {
+    super(config, logger);
+    
+    // Tool metadata
+    this.requiresProject = false;
+    this.isAsync = true; // Browser operations can be slow
+    this.timeout = config.timeout || 60000; // 1 minute default
+    this.maxConcurrentOperations = config.maxConcurrentOperations || 2;
+    
+    // Browser configuration
+    this.browserConfig = {
+      headless: config.headless !== false, // Default to headless
+      viewport: config.viewport || { width: 1280, height: 720 },
+      userAgent: config.userAgent || getDefaultUserAgent(),
+      timeout: config.pageTimeout || 30000
+    };
+    
+    // Active browser instances
+    this.browserInstances = new Map();
+    
+    // Security settings
+    this.allowedDomains = config.allowedDomains || null; // null = all allowed
+    this.blockedDomains = config.blockedDomains || [];
+    this.maxScreenshotSize = config.maxScreenshotSize || 5 * 1024 * 1024; // 5MB
+    
+    // Operation history
+    this.operationHistory = [];
+  }
+
+  /**
+   * Get tool description for LLM consumption
+   * @returns {string} Tool description
+   */
+  getDescription() {
+    return `
+Browser Tool: Automate web browser interactions, navigation, and data extraction.
+
+USAGE:
+\`\`\`json
+{
+  "toolId": "browser",
+  "actions": [
+    {"type": "navigate", "url": "https://example.com"},
+    {"type": "wait", "waitTime": 2000},
+    {"type": "click", "selector": "button.submit"},
+    {"type": "screenshot", "outputPath": "screenshot.png"}
+  ]
+}
+\`\`\`
+
+SUPPORTED ACTIONS:
+- navigate: Navigate to URL (url)
+- click: Click element (selector)
+- type: Type text (selector, text)
+- wait: Wait for element/time (waitTime or waitSelector)
+- screenshot: Take screenshot (outputPath, fullPage)
+- extract: Extract data (selector, attribute)
+- scroll: Scroll page (direction, pixels)
+- close: Close browser
+
+PARAMETERS:
+- url: Target URL for navigation
+- selector: CSS selector (e.g., "#id", ".class", "button[type='submit']")
+- text: Text to type
+- outputPath: Screenshot file path
+- waitTime: Wait in milliseconds (100-30000)
+- waitSelector: Element selector to wait for
+- attribute: HTML attribute to extract (default: "text")
+- direction: "up", "down", "left", "right"
+- pixels: Scroll distance (default: 300)
+- fullPage: Full page screenshot (true/false)
+
+EXAMPLES:
+
+Screenshot a page:
+\`\`\`json
+{
+  "toolId": "browser",
+  "actions": [
+    {"type": "navigate", "url": "https://github.com/trending"},
+    {"type": "wait", "waitTime": 2000},
+    {"type": "screenshot", "outputPath": "trending.png"},
+    {"type": "close"}
+  ]
+}
+\`\`\`
+
+Form automation:
+\`\`\`json
+{
+  "toolId": "browser",
+  "actions": [
+    {"type": "navigate", "url": "https://example.com/login"},
+    {"type": "wait", "waitSelector": "input#username"},
+    {"type": "type", "selector": "input#username", "text": "myuser"},
+    {"type": "type", "selector": "input#password", "text": "mypass"},
+    {"type": "click", "selector": "button[type='submit']"},
+    {"type": "close"}
+  ]
+}
+\`\`\`
+
+SECURITY:
+- Restricted to allowed domains
+- Screenshot size limits (max 5MB)
+- Auto-cleanup after 10 minutes
+
+VIEWPORT: ${this.browserConfig.viewport.width}x${this.browserConfig.viewport.height}
+TIMEOUT: ${this.browserConfig.timeout}ms per operation
+    `;
+  }
+
+  /**
+   * Parse parameters from tool command content
+   * @param {string} content - Raw tool command content
+   * @returns {Object} Parsed parameters
+   */
+  parseParameters(content) {
+    try {
+      // Try JSON first (for JSON in code blocks)
+      if (content.trim().startsWith('{')) {
+        const parsed = JSON.parse(content);
+        // Ensure actions is an array
+        if (!parsed.actions) {
+          parsed.actions = [];
+        }
+        if (!Array.isArray(parsed.actions)) {
+          parsed.actions = [parsed.actions];
+        }
+        return parsed;
+      }
+
+      const params = {};
+      const actions = [];
+
+      // Extract self-closing tags (e.g., <navigate url="..." />)
+      const selfClosingTags = [
+        'navigate', 'click', 'type', 'wait', 'screenshot',
+        'extract', 'scroll', 'close'
+      ];
+
+      for (const tagName of selfClosingTags) {
+        // Extract self-closing tags using regex
+        const selfClosingPattern = new RegExp(
+          `<${tagName}\\s+([^>]*?)\\s*\\/?>`,
+          'gi'
+        );
+
+        let match;
+        while ((match = selfClosingPattern.exec(content)) !== null) {
+          const attributeString = match[1];
+          const attributes = this.parseAttributes(attributeString);
+
+          const action = {
+            type: tagName,
+            ...attributes
+          };
+
+          // Normalize common attribute names (kebab-case to camelCase)
+          if (action['output-path']) {
+            action.outputPath = action['output-path'];
+            delete action['output-path'];
+          }
+          if (action['wait-time']) {
+            action.waitTime = parseInt(action['wait-time'], 10);
+            delete action['wait-time'];
+          }
+          if (action['wait-selector']) {
+            action.waitSelector = action['wait-selector'];
+            delete action['wait-selector'];
+          }
+          if (action['wait-type']) {
+            action.waitType = action['wait-type'];
+            delete action['wait-type'];
+          }
+          if (action['full-page']) {
+            action.fullPage = action['full-page'] === 'true';
+            delete action['full-page'];
+          }
+
+          actions.push(action);
+        }
+      }
+
+      // Ensure actions is always an array (even if empty)
+      params.actions = Array.isArray(actions) ? actions : [];
+      params.rawContent = content.trim();
+
+      return params;
+
+    } catch (error) {
+      // If JSON parsing fails, return a structure with empty actions
+      // This prevents "actions is not iterable" errors
+      this.logger?.error('Failed to parse browser parameters', { error: error.message });
+      return {
+        actions: [], // Always return an array
+        rawContent: content.trim(),
+        parseError: error.message
+      };
+    }
+  }
+
+  /**
+   * Parse attributes from attribute string
+   * @param {string} attributeString - String containing attributes
+   * @returns {Object} Parsed attributes object
+   * @private
+   */
+  parseAttributes(attributeString) {
+    const attributes = {};
+
+    if (!attributeString) return attributes;
+
+    // Match attribute="value" or attribute='value'
+    const attrPattern = /([\w-]+)\s*=\s*["']([^"']*)["']/g;
+    let match;
+
+    while ((match = attrPattern.exec(attributeString)) !== null) {
+      const attrName = match[1];
+      const attrValue = match[2];
+      attributes[attrName] = attrValue;
+    }
+
+    return attributes;
+  }
+
+  /**
+   * Get required parameters
+   * @returns {Array<string>} Array of required parameter names
+   */
+  getRequiredParameters() {
+    return ['actions'];
+  }
+
+  /**
+   * Custom parameter validation
+   * @param {Object} params - Parameters to validate
+   * @returns {Object} Validation result
+   */
+  customValidateParameters(params) {
+    const errors = [];
+    
+    if (!params.actions || !Array.isArray(params.actions) || params.actions.length === 0) {
+      errors.push('At least one action is required');
+    } else {
+      // Validate each action
+      for (const [index, action] of params.actions.entries()) {
+        if (!action.type) {
+          errors.push(`Action ${index + 1}: type is required`);
+          continue;
+        }
+        
+        switch (action.type) {
+          case 'navigate':
+            if (!action.url) {
+              errors.push(`Action ${index + 1}: url is required for navigate`);
+            } else if (!this.isAllowedUrl(action.url)) {
+              errors.push(`Action ${index + 1}: URL not allowed: ${action.url}`);
+            }
+            break;
+            
+          case 'click':
+          case 'extract':
+            if (!action.selector) {
+              errors.push(`Action ${index + 1}: selector is required for ${action.type}`);
+            }
+            break;
+            
+          case 'type':
+            if (!action.selector) {
+              errors.push(`Action ${index + 1}: selector is required for type`);
+            }
+            if (!action.text && action.text !== '') {
+              errors.push(`Action ${index + 1}: text is required for type`);
+            }
+            break;
+            
+          case 'wait':
+            if (!action.waitTime && !action.waitSelector) {
+              errors.push(`Action ${index + 1}: either wait-time or wait-selector is required`);
+            }
+            if (action.waitTime && (action.waitTime < 100 || action.waitTime > 30000)) {
+              errors.push(`Action ${index + 1}: wait-time must be between 100 and 30000 milliseconds`);
+            }
+            break;
+            
+          case 'screenshot':
+            if (!action.outputPath) {
+              errors.push(`Action ${index + 1}: output-path is required for screenshot`);
+            }
+            break;
+            
+          case 'scroll':
+            if (!action.direction) {
+              errors.push(`Action ${index + 1}: direction is required for scroll`);
+            } else if (!['up', 'down', 'left', 'right'].includes(action.direction)) {
+              errors.push(`Action ${index + 1}: invalid scroll direction: ${action.direction}`);
+            }
+            break;
+            
+          case 'close':
+            // No additional validation needed
+            break;
+            
+          default:
+            errors.push(`Action ${index + 1}: unknown action type: ${action.type}`);
+        }
+      }
+    }
+    
+    return {
+      valid: errors.length === 0,
+      errors
+    };
+  }
+
+  /**
+   * Execute tool with parsed parameters
+   * @param {Object} params - Parsed parameters
+   * @param {Object} context - Execution context
+   * @returns {Promise<Object>} Execution result
+   */
+  async execute(params, context) {
+    const { actions } = params;
+    const { agentId, projectDir } = context;
+    
+    const browserKey = `${agentId}-${projectDir || 'default'}`;
+    const results = [];
+    
+    for (const action of actions) {
+      try {
+        let result;
+        
+        switch (action.type) {
+          case 'navigate':
+            result = await this.navigate(action.url, browserKey);
+            break;
+            
+          case 'click':
+            result = await this.click(action.selector, browserKey);
+            break;
+            
+          case 'type':
+            result = await this.type(action.selector, action.text, browserKey);
+            break;
+            
+          case 'wait':
+            result = await this.wait(action, browserKey);
+            break;
+            
+          case 'screenshot':
+            result = await this.screenshot(action.outputPath, projectDir, browserKey, action);
+            break;
+            
+          case 'extract':
+            result = await this.extract(action.selector, action.attribute || 'text', browserKey);
+            break;
+            
+          case 'scroll':
+            result = await this.scroll(action, browserKey);
+            break;
+            
+          case 'close':
+            result = await this.closeBrowser(browserKey);
+            break;
+            
+          default:
+            throw new Error(`Unknown action type: ${action.type}`);
+        }
+        
+        results.push(result);
+        this.addToHistory(action, result, agentId);
+        
+      } catch (error) {
+        const errorResult = {
+          success: false,
+          action: action.type,
+          error: error.message,
+          selector: action.selector || action.url
+        };
+        
+        results.push(errorResult);
+        this.addToHistory(action, errorResult, agentId);
+      }
+    }
+    
+    return {
+      success: true,
+      actions: results,
+      executedActions: actions.length,
+      toolUsed: 'browser'
+    };
+  }
+
+  /**
+   * Navigate to URL
+   * @private
+   */
+  async navigate(url, browserKey) {
+    const browser = await this.getBrowser(browserKey);
+    const page = await this.getPage(browserKey);
+    
+    const startTime = Date.now();
+    
+    try {
+      await page.goto(url, { 
+        waitUntil: 'networkidle0', 
+        timeout: this.browserConfig.timeout 
+      });
+      
+      const title = await page.title();
+      const currentUrl = page.url();
+      
+      return {
+        success: true,
+        action: 'navigate',
+        url: currentUrl,
+        originalUrl: url,
+        title,
+        loadTime: Date.now() - startTime,
+        message: `Navigated to ${url}`
+      };
+      
+    } catch (error) {
+      throw new Error(`Navigation failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Click on element
+   * @private
+   */
+  async click(selector, browserKey) {
+    const page = await this.getPage(browserKey);
+    
+    try {
+      await page.waitForSelector(selector, { timeout: 5000 });
+      await page.click(selector);
+      
+      return {
+        success: true,
+        action: 'click',
+        selector,
+        message: `Clicked on ${selector}`
+      };
+      
+    } catch (error) {
+      throw new Error(`Click failed on ${selector}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Type text into element
+   * @private
+   */
+  async type(selector, text, browserKey) {
+    const page = await this.getPage(browserKey);
+    
+    try {
+      await page.waitForSelector(selector, { timeout: 5000 });
+      await page.focus(selector);
+      await page.keyboard.selectAll();
+      await page.type(selector, text);
+      
+      return {
+        success: true,
+        action: 'type',
+        selector,
+        text,
+        textLength: text.length,
+        message: `Typed ${text.length} characters into ${selector}`
+      };
+      
+    } catch (error) {
+      throw new Error(`Type failed on ${selector}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Wait for condition
+   * @private
+   */
+  async wait(action, browserKey) {
+    const page = await this.getPage(browserKey);
+
+    try {
+      if (action.waitTime) {
+        // Use setTimeout wrapped in Promise (waitForTimeout was removed in Puppeteer v22+)
+        await new Promise(resolve => setTimeout(resolve, action.waitTime));
+        return {
+          success: true,
+          action: 'wait',
+          waitTime: action.waitTime,
+          message: `Waited ${action.waitTime}ms`
+        };
+      }
+
+      if (action.waitSelector) {
+        const waitType = action.waitType || 'visible';
+        const options = { timeout: 10000 };
+
+        if (waitType === 'hidden') {
+          await page.waitForSelector(action.waitSelector, { ...options, hidden: true });
+        } else {
+          await page.waitForSelector(action.waitSelector, options);
+        }
+
+        return {
+          success: true,
+          action: 'wait',
+          waitSelector: action.waitSelector,
+          waitType,
+          message: `Waited for ${action.waitSelector} to be ${waitType}`
+        };
+      }
+
+    } catch (error) {
+      throw new Error(`Wait failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Take screenshot
+   * @private
+   */
+  async screenshot(outputPath, projectDir, browserKey, options = {}) {
+    const page = await this.getPage(browserKey);
+    
+    try {
+      const fullPath = projectDir ? path.resolve(projectDir, outputPath) : path.resolve(outputPath);
+      
+      // Ensure directory exists
+      const dir = path.dirname(fullPath);
+      await fs.mkdir(dir, { recursive: true });
+      
+      const screenshotOptions = {
+        path: fullPath,
+        type: 'png'
+      };
+      
+      if (options.fullPage) {
+        screenshotOptions.fullPage = true;
+      }
+      
+      await page.screenshot(screenshotOptions);
+      
+      const stats = await fs.stat(fullPath);
+      
+      if (stats.size > this.maxScreenshotSize) {
+        throw new Error(`Screenshot too large: ${stats.size} bytes (max ${this.maxScreenshotSize})`);
+      }
+      
+      return {
+        success: true,
+        action: 'screenshot',
+        outputPath: path.relative(projectDir || process.cwd(), fullPath),
+        size: stats.size,
+        fullPage: !!options.fullPage,
+        message: `Screenshot saved: ${outputPath}`
+      };
+      
+    } catch (error) {
+      throw new Error(`Screenshot failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Extract data from elements
+   * @private
+   */
+  async extract(selector, attribute, browserKey) {
+    const page = await this.getPage(browserKey);
+    
+    try {
+      await page.waitForSelector(selector, { timeout: 5000 });
+      
+      let data;
+      if (attribute === 'text') {
+        data = await page.$eval(selector, el => el.textContent.trim());
+      } else {
+        data = await page.$eval(selector, (el, attr) => el.getAttribute(attr), attribute);
+      }
+      
+      return {
+        success: true,
+        action: 'extract',
+        selector,
+        attribute,
+        data,
+        message: `Extracted ${attribute} from ${selector}`
+      };
+      
+    } catch (error) {
+      throw new Error(`Extract failed on ${selector}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Scroll page or element
+   * @private
+   */
+  async scroll(action, browserKey) {
+    const page = await this.getPage(browserKey);
+    
+    try {
+      const { direction, pixels = 300, selector } = action;
+      
+      if (selector) {
+        // Scroll specific element
+        await page.evaluate((sel, dir, px) => {
+          const element = document.querySelector(sel);
+          if (element) {
+            if (dir === 'down') element.scrollTop += px;
+            else if (dir === 'up') element.scrollTop -= px;
+            else if (dir === 'right') element.scrollLeft += px;
+            else if (dir === 'left') element.scrollLeft -= px;
+          }
+        }, selector, direction, pixels);
+      } else {
+        // Scroll page
+        await page.evaluate((dir, px) => {
+          if (dir === 'down') window.scrollBy(0, px);
+          else if (dir === 'up') window.scrollBy(0, -px);
+          else if (dir === 'right') window.scrollBy(px, 0);
+          else if (dir === 'left') window.scrollBy(-px, 0);
+        }, direction, pixels);
+      }
+      
+      return {
+        success: true,
+        action: 'scroll',
+        direction,
+        pixels,
+        selector: selector || 'page',
+        message: `Scrolled ${direction} by ${pixels}px`
+      };
+      
+    } catch (error) {
+      throw new Error(`Scroll failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Get or create browser instance
+   * @private
+   */
+  async getBrowser(browserKey) {
+    if (this.browserInstances.has(browserKey)) {
+      const instance = this.browserInstances.get(browserKey);
+      if (!instance.browser.isConnected()) {
+        this.browserInstances.delete(browserKey);
+      } else {
+        return instance.browser;
+      }
+    }
+    
+    const browser = await puppeteer.launch({
+      headless: this.browserConfig.headless,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-gpu'
+      ]
+    });
+    
+    const page = await browser.newPage();
+    await page.setViewport(this.browserConfig.viewport);
+    await page.setUserAgent(this.browserConfig.userAgent);
+    
+    this.browserInstances.set(browserKey, {
+      browser,
+      page,
+      created: Date.now()
+    });
+    
+    // Auto-cleanup after 10 minutes of inactivity
+    setTimeout(() => {
+      this.cleanupBrowser(browserKey);
+    }, 10 * 60 * 1000);
+    
+    return browser;
+  }
+
+  /**
+   * Get page instance
+   * @private
+   */
+  async getPage(browserKey) {
+    await this.getBrowser(browserKey);
+    return this.browserInstances.get(browserKey).page;
+  }
+
+  /**
+   * Close browser instance
+   * @private
+   */
+  async closeBrowser(browserKey) {
+    const instance = this.browserInstances.get(browserKey);
+    
+    if (instance) {
+      try {
+        await instance.browser.close();
+        this.browserInstances.delete(browserKey);
+        
+        return {
+          success: true,
+          action: 'close',
+          browserKey,
+          message: 'Browser instance closed'
+        };
+      } catch (error) {
+        throw new Error(`Failed to close browser: ${error.message}`);
+      }
+    }
+    
+    return {
+      success: true,
+      action: 'close',
+      browserKey,
+      message: 'Browser instance not found (already closed)'
+    };
+  }
+
+  /**
+   * Cleanup browser instance
+   * @private
+   */
+  async cleanupBrowser(browserKey) {
+    const instance = this.browserInstances.get(browserKey);
+    
+    if (instance) {
+      try {
+        await instance.browser.close();
+      } catch (error) {
+        // Ignore cleanup errors
+      } finally {
+        this.browserInstances.delete(browserKey);
+      }
+    }
+  }
+
+  /**
+   * Check if URL is allowed
+   * @private
+   */
+  isAllowedUrl(url) {
+    try {
+      const urlObj = new URL(url);
+      const hostname = urlObj.hostname.toLowerCase();
+      
+      // Check blocked domains
+      if (this.blockedDomains.some(blocked => hostname.includes(blocked.toLowerCase()))) {
+        return false;
+      }
+      
+      // Check allowed domains if specified
+      if (this.allowedDomains && this.allowedDomains.length > 0) {
+        return this.allowedDomains.some(allowed => hostname.includes(allowed.toLowerCase()));
+      }
+      
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Add operation to history
+   * @private
+   */
+  addToHistory(action, result, agentId) {
+    const historyEntry = {
+      timestamp: new Date().toISOString(),
+      agentId,
+      action: action.type,
+      selector: action.selector || action.url,
+      success: result.success,
+      loadTime: result.loadTime || 0
+    };
+    
+    this.operationHistory.push(historyEntry);
+    
+    // Keep only last 100 entries
+    if (this.operationHistory.length > 100) {
+      this.operationHistory = this.operationHistory.slice(-100);
+    }
+  }
+
+  /**
+   * Get supported actions for this tool
+   * @returns {Array<string>} Array of supported action names
+   */
+  getSupportedActions() {
+    return ['navigate', 'click', 'type', 'wait', 'screenshot', 'extract', 'scroll', 'close'];
+  }
+
+  /**
+   * Get parameter schema for validation
+   * @returns {Object} Parameter schema
+   */
+  getParameterSchema() {
+    return {
+      type: 'object',
+      properties: {
+        actions: {
+          type: 'array',
+          minItems: 1,
+          items: {
+            type: 'object',
+            properties: {
+              type: {
+                type: 'string',
+                enum: this.getSupportedActions()
+              },
+              url: { type: 'string' },
+              selector: { type: 'string' },
+              text: { type: 'string' },
+              outputPath: { type: 'string' },
+              waitTime: { type: 'integer' },
+              waitSelector: { type: 'string' },
+              attribute: { type: 'string' },
+              direction: { type: 'string' },
+              pixels: { type: 'integer' }
+            },
+            required: ['type']
+          }
+        }
+      },
+      required: ['actions']
+    };
+  }
+
+  /**
+   * Resource cleanup
+   * @param {string} operationId - Operation identifier
+   */
+  async cleanup(operationId) {
+    // Close all browser instances
+    for (const [browserKey] of this.browserInstances) {
+      await this.cleanupBrowser(browserKey);
+    }
+  }
+
+  /**
+   * Get operation history for debugging
+   * @returns {Array} Operation history
+   */
+  getOperationHistory(agentId = null) {
+    if (agentId) {
+      return this.operationHistory.filter(entry => entry.agentId === agentId);
+    }
+    return [...this.operationHistory];
+  }
+}
+
+export default BrowserTool;