musubi-sdd 2.2.0 → 3.0.0

package/src/agents/browser/index.js

@@ -0,0 +1,265 @@
+/**
+ * @fileoverview Browser Agent - Natural Language Browser Automation
+ * @module agents/browser
+ * @description Provides browser automation through natural language commands,
+ * screenshot capture and comparison, and E2E test generation.
+ */
+
+const { chromium, firefox, webkit } = require('playwright');
+const NLParser = require('./nl-parser');
+const ActionExecutor = require('./action-executor');
+const ContextManager = require('./context-manager');
+const ScreenshotCapture = require('./screenshot');
+const AIComparator = require('./ai-comparator');
+const TestGenerator = require('./test-generator');
+
+/**
+ * @typedef {Object} BrowserAgentOptions
+ * @property {boolean} [headless=true] - Run browser in headless mode
+ * @property {'chromium'|'firefox'|'webkit'} [browser='chromium'] - Browser type
+ * @property {string} [outputDir='./screenshots'] - Screenshot output directory
+ * @property {string} [visionModel='gpt-4-vision-preview'] - Vision AI model
+ * @property {number} [timeout=30000] - Default timeout in milliseconds
+ * @property {number} [threshold=0.95] - Screenshot comparison threshold
+ */
+
+/**
+ * Browser Agent for natural language browser automation
+ */
+class BrowserAgent {
+  /**
+   * Create a new BrowserAgent instance
+   * @param {BrowserAgentOptions} options - Configuration options
+   */
+  constructor(options = {}) {
+    this.options = {
+      headless: options.headless !== false,
+      browser: options.browser || 'chromium',
+      outputDir: options.outputDir || './screenshots',
+      visionModel: options.visionModel || 'gpt-4-vision-preview',
+      timeout: options.timeout || 30000,
+      threshold: options.threshold || 0.95,
+    };
+
+    this.parser = new NLParser();
+    this.executor = new ActionExecutor();
+    this.contextManager = new ContextManager();
+    this.screenshot = new ScreenshotCapture(this.options.outputDir);
+    this.comparator = new AIComparator({
+      model: this.options.visionModel,
+      threshold: this.options.threshold,
+    });
+    this.testGenerator = new TestGenerator();
+
+    this.browser = null;
+    this.isLaunched = false;
+  }
+
+  /**
+   * Launch the browser
+   * @returns {Promise<void>}
+   */
+  async launch() {
+    if (this.isLaunched) {
+      return;
+    }
+
+    const browserType = this.getBrowserType();
+    this.browser = await browserType.launch({
+      headless: this.options.headless,
+    });
+
+    await this.contextManager.initialize(this.browser);
+    this.isLaunched = true;
+  }
+
+  /**
+   * Get the Playwright browser type
+   * @returns {import('playwright').BrowserType}
+   * @private
+   */
+  getBrowserType() {
+    switch (this.options.browser) {
+      case 'firefox':
+        return firefox;
+      case 'webkit':
+        return webkit;
+      case 'chromium':
+      default:
+        return chromium;
+    }
+  }
+
+  /**
+   * Execute a natural language command
+   * @param {string} command - Natural language command
+   * @returns {Promise<import('./action-executor').ActionResult>}
+   */
+  async execute(command) {
+    if (!this.isLaunched) {
+      await this.launch();
+    }
+
+    // Parse natural language to actions
+    const parseResult = this.parser.parse(command);
+    
+    if (!parseResult.success) {
+      return {
+        success: false,
+        error: parseResult.error,
+        command,
+      };
+    }
+
+    // Get current page context
+    const page = await this.contextManager.getOrCreatePage();
+
+    // Execute each action
+    const results = [];
+    for (const action of parseResult.actions) {
+      const result = await this.executor.execute(action, {
+        page,
+        screenshot: this.screenshot,
+        timeout: this.options.timeout,
+      });
+      
+      results.push(result);
+      
+      if (!result.success) {
+        return {
+          success: false,
+          error: result.error,
+          action,
+          results,
+        };
+      }
+    }
+
+    return {
+      success: true,
+      command,
+      actions: parseResult.actions,
+      results,
+    };
+  }
+
+  /**
+   * Execute multiple commands in sequence
+   * @param {string[]} commands - Array of natural language commands
+   * @returns {Promise<Object>}
+   */
+  async executeSequence(commands) {
+    const results = [];
+    
+    for (const command of commands) {
+      const result = await this.execute(command);
+      results.push(result);
+      
+      if (!result.success) {
+        return {
+          success: false,
+          error: result.error,
+          completedCommands: results.length - 1,
+          results,
+        };
+      }
+    }
+
+    return {
+      success: true,
+      totalCommands: commands.length,
+      results,
+    };
+  }
+
+  /**
+   * Take a screenshot of the current page
+   * @param {Object} options - Screenshot options
+   * @param {string} [options.name] - Screenshot name
+   * @param {boolean} [options.fullPage=false] - Capture full page
+   * @returns {Promise<string>} Screenshot file path
+   */
+  async takeScreenshot(options = {}) {
+    if (!this.isLaunched) {
+      throw new Error('Browser not launched. Call launch() first.');
+    }
+
+    const page = await this.contextManager.getOrCreatePage();
+    return this.screenshot.capture(page, options);
+  }
+
+  /**
+   * Compare two screenshots using AI
+   * @param {string} expected - Path to expected screenshot
+   * @param {string} actual - Path to actual screenshot
+   * @param {Object} options - Comparison options
+   * @param {number} [options.threshold] - Similarity threshold
+   * @param {string} [options.description] - What to verify
+   * @returns {Promise<import('./ai-comparator').ComparisonResult>}
+   */
+  async compare(expected, actual, options = {}) {
+    return this.comparator.compare(expected, actual, {
+      threshold: options.threshold || this.options.threshold,
+      description: options.description,
+    });
+  }
+
+  /**
+   * Generate Playwright test code from action history
+   * @param {Object} options - Generation options
+   * @param {string} [options.name] - Test name
+   * @param {string} [options.output] - Output file path
+   * @returns {Promise<string>} Generated test code
+   */
+  async generateTest(options = {}) {
+    const history = this.contextManager.getActionHistory();
+    return this.testGenerator.generateTest(history, options);
+  }
+
+  /**
+   * Get the current page
+   * @returns {Promise<import('playwright').Page>}
+   */
+  async getPage() {
+    if (!this.isLaunched) {
+      await this.launch();
+    }
+    return this.contextManager.getOrCreatePage();
+  }
+
+  /**
+   * Get action history
+   * @returns {Array<import('./action-executor').Action>}
+   */
+  getActionHistory() {
+    return this.contextManager.getActionHistory();
+  }
+
+  /**
+   * Clear action history
+   */
+  clearHistory() {
+    this.contextManager.clearHistory();
+  }
+
+  /**
+   * Close the browser
+   * @returns {Promise<void>}
+   */
+  async close() {
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+      this.isLaunched = false;
+      this.contextManager.reset();
+    }
+  }
+}
+
+module.exports = BrowserAgent;
+module.exports.NLParser = NLParser;
+module.exports.ActionExecutor = ActionExecutor;
+module.exports.ContextManager = ContextManager;
+module.exports.ScreenshotCapture = ScreenshotCapture;
+module.exports.AIComparator = AIComparator;
+module.exports.TestGenerator = TestGenerator;

package/src/agents/browser/nl-parser.js

@@ -0,0 +1,408 @@
+/**
+ * @fileoverview Natural Language Parser for Browser Commands
+ * @module agents/browser/nl-parser
+ */
+
+/**
+ * @typedef {Object} Action
+ * @property {string} type - Action type
+ * @property {string} [url] - URL for navigate actions
+ * @property {string} [selector] - CSS selector for element actions
+ * @property {string} [value] - Value for fill/select actions
+ * @property {number} [delay] - Delay in milliseconds for wait actions
+ * @property {string} [name] - Name for screenshot actions
+ * @property {boolean} [fullPage] - Full page screenshot flag
+ */
+
+/**
+ * @typedef {Object} ParseResult
+ * @property {boolean} success - Whether parsing succeeded
+ * @property {Action[]} actions - Parsed actions
+ * @property {string} [error] - Error message if failed
+ */
+
+/**
+ * Action patterns for Japanese and English
+ */
+const ACTION_PATTERNS = {
+  navigate: {
+    patterns: [
+      /(?:に)?移動|開く|アクセス/,
+      /(?:go to|navigate|open|visit)/i,
+    ],
+    urlPattern: /(https?:\/\/[^\s]+)/,
+  },
+  click: {
+    patterns: [
+      /クリック|押す|タップ|選択/,
+      /click|press|tap|select/i,
+    ],
+  },
+  fill: {
+    patterns: [
+      /(?:に|を)?入力|記入|タイプ/,
+      /fill|type|enter|input/i,
+    ],
+    valuePattern: /[「「]([^」」]+)[」」]|"([^"]+)"|'([^']+)'/,
+  },
+  select: {
+    patterns: [
+      /ドロップダウン.*選択|選択.*オプション/,
+      /select.*dropdown|choose.*option/i,
+    ],
+  },
+  wait: {
+    patterns: [
+      /秒?待つ|待機/,
+      /wait|pause|delay/i,
+    ],
+    durationPattern: /(\d+)\s*秒|(\d+)\s*(?:seconds?|ms|milliseconds?)/i,
+  },
+  screenshot: {
+    patterns: [
+      /スクリーンショット|画面.*(?:保存|撮|キャプチャ)/,
+      /screenshot|capture|save.*screen/i,
+    ],
+    namePattern: /[「「]([^」」]+)[」」]|"([^"]+)"|として\s*(\S+)/,
+  },
+  assert: {
+    patterns: [
+      /(?:が)?表示|確認|検証|存在/,
+      /(?:is )?visible|assert|verify|check|exists?/i,
+    ],
+    textPattern: /[「「]([^」」]+)[」」]|"([^"]+)"/,
+  },
+};
+
+/**
+ * Common element selector patterns
+ */
+const ELEMENT_PATTERNS = {
+  // Japanese element names
+  'ログインボタン': 'button:has-text("ログイン"), [data-testid="login-button"], button[type="submit"]',
+  '送信ボタン': 'button:has-text("送信"), [data-testid="submit-button"], button[type="submit"]',
+  'メール': 'input[type="email"], input[name="email"], [data-testid="email-input"]',
+  'パスワード': 'input[type="password"], [data-testid="password-input"]',
+  '検索': 'input[type="search"], [data-testid="search-input"], input[name="q"]',
+  // English element names
+  'login button': 'button:has-text("Login"), [data-testid="login-button"], button[type="submit"]',
+  'submit button': 'button:has-text("Submit"), [data-testid="submit-button"], button[type="submit"]',
+  'email': 'input[type="email"], input[name="email"], [data-testid="email-input"]',
+  'password': 'input[type="password"], [data-testid="password-input"]',
+  'search': 'input[type="search"], [data-testid="search-input"], input[name="q"]',
+};
+
+/**
+ * Natural Language Parser for browser commands
+ */
+class NLParser {
+  constructor() {
+    this.actionPatterns = ACTION_PATTERNS;
+    this.elementPatterns = ELEMENT_PATTERNS;
+  }
+
+  /**
+   * Parse a natural language command into actions
+   * @param {string} command - Natural language command
+   * @returns {ParseResult}
+   */
+  parse(command) {
+    try {
+      const normalizedCommand = this.normalizeCommand(command);
+      const actions = this.extractActions(normalizedCommand);
+
+      if (actions.length === 0) {
+        return {
+          success: false,
+          actions: [],
+          error: `Could not understand command: "${command}"`,
+        };
+      }
+
+      return {
+        success: true,
+        actions,
+      };
+    } catch (error) {
+      return {
+        success: false,
+        actions: [],
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Normalize a command for parsing
+   * @param {string} command
+   * @returns {string}
+   */
+  normalizeCommand(command) {
+    return command
+      .trim()
+      .replace(/\s+/g, ' ')
+      .replace(/、/g, ',')
+      .replace(/。/g, '.');
+  }
+
+  /**
+   * Extract actions from a normalized command
+   * @param {string} command
+   * @returns {Action[]}
+   */
+  extractActions(command) {
+    const actions = [];
+    const parts = this.splitCommand(command);
+
+    for (const part of parts) {
+      const action = this.parseAction(part);
+      if (action) {
+        actions.push(action);
+      }
+    }
+
+    return actions;
+  }
+
+  /**
+   * Split command into individual action parts
+   * @param {string} command
+   * @returns {string[]}
+   */
+  splitCommand(command) {
+    // Split by conjunctions and separators
+    const separators = /[,、]|\s+(?:そして|して|and|then)\s+/i;
+    return command.split(separators).map(s => s.trim()).filter(Boolean);
+  }
+
+  /**
+   * Parse a single action from text
+   * @param {string} text
+   * @returns {Action|null}
+   */
+  parseAction(text) {
+    // Try each action type
+    if (this.matchesPattern(text, this.actionPatterns.navigate.patterns)) {
+      return this.parseNavigate(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.screenshot.patterns)) {
+      return this.parseScreenshot(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.wait.patterns)) {
+      return this.parseWait(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.fill.patterns)) {
+      return this.parseFill(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.select.patterns)) {
+      return this.parseSelect(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.click.patterns)) {
+      return this.parseClick(text);
+    }
+
+    if (this.matchesPattern(text, this.actionPatterns.assert.patterns)) {
+      return this.parseAssert(text);
+    }
+
+    return null;
+  }
+
+  /**
+   * Check if text matches any of the patterns
+   * @param {string} text
+   * @param {RegExp[]} patterns
+   * @returns {boolean}
+   */
+  matchesPattern(text, patterns) {
+    return patterns.some(pattern => pattern.test(text));
+  }
+
+  /**
+   * Parse navigate action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseNavigate(text) {
+    const urlMatch = text.match(this.actionPatterns.navigate.urlPattern);
+    const url = urlMatch ? urlMatch[1] : null;
+
+    return {
+      type: 'navigate',
+      url: url || 'about:blank',
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse click action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseClick(text) {
+    const selector = this.extractSelector(text);
+
+    return {
+      type: 'click',
+      selector,
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse fill action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseFill(text) {
+    const selector = this.extractSelector(text);
+    const valueMatch = text.match(this.actionPatterns.fill.valuePattern);
+    const value = valueMatch ? (valueMatch[1] || valueMatch[2] || valueMatch[3]) : '';
+
+    return {
+      type: 'fill',
+      selector,
+      value,
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse select action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseSelect(text) {
+    const selector = this.extractSelector(text);
+    const valueMatch = text.match(this.actionPatterns.fill.valuePattern);
+    const value = valueMatch ? (valueMatch[1] || valueMatch[2] || valueMatch[3]) : '';
+
+    return {
+      type: 'select',
+      selector,
+      value,
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse wait action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseWait(text) {
+    const durationMatch = text.match(this.actionPatterns.wait.durationPattern);
+    let delay = 1000; // Default 1 second
+
+    if (durationMatch) {
+      const seconds = durationMatch[1] || durationMatch[2];
+      delay = parseInt(seconds, 10) * 1000;
+      
+      // Check if it's milliseconds
+      if (/ms|milliseconds?/i.test(text)) {
+        delay = parseInt(seconds, 10);
+      }
+    }
+
+    return {
+      type: 'wait',
+      delay,
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse screenshot action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseScreenshot(text) {
+    const nameMatch = text.match(this.actionPatterns.screenshot.namePattern);
+    const name = nameMatch ? (nameMatch[1] || nameMatch[2] || nameMatch[3]) : undefined;
+    const fullPage = /全体|full\s*page/i.test(text);
+
+    return {
+      type: 'screenshot',
+      name,
+      fullPage,
+      raw: text,
+    };
+  }
+
+  /**
+   * Parse assert action
+   * @param {string} text
+   * @returns {Action}
+   */
+  parseAssert(text) {
+    const textMatch = text.match(this.actionPatterns.assert.textPattern);
+    const expectedText = textMatch ? (textMatch[1] || textMatch[2]) : null;
+    const selector = expectedText ? `text="${expectedText}"` : this.extractSelector(text);
+
+    return {
+      type: 'assert',
+      selector,
+      expectedText,
+      raw: text,
+    };
+  }
+
+  /**
+   * Extract a CSS selector from text
+   * @param {string} text
+   * @returns {string}
+   */
+  extractSelector(text) {
+    // Check for known element patterns
+    for (const [name, selector] of Object.entries(this.elementPatterns)) {
+      if (text.toLowerCase().includes(name.toLowerCase())) {
+        return selector;
+      }
+    }
+
+    // Check for CSS selector in the text (match only the selector part)
+    const selectorMatch = text.match(/([#\.][a-zA-Z][\w\-]*|\[[^\]]+\])/);
+    if (selectorMatch) {
+      return selectorMatch[0];
+    }
+
+    // Check for data-testid
+    const testIdMatch = text.match(/data-testid[=:]["']?([^"'\s]+)/i);
+    if (testIdMatch) {
+      return `[data-testid="${testIdMatch[1]}"]`;
+    }
+
+    // Try to extract element description
+    const elementMatch = text.match(/(?:の)?(?:ボタン|リンク|入力欄?|フィールド|テキスト|button|link|input|field|text)\s*[「「]?([^」」\s]*)[」」]?/i);
+    if (elementMatch && elementMatch[1]) {
+      return `text="${elementMatch[1]}"`;
+    }
+
+    // Default to a generic selector
+    return '*';
+  }
+
+  /**
+   * Add a custom element pattern
+   * @param {string} name - Element name
+   * @param {string} selector - CSS selector
+   */
+  addElementPattern(name, selector) {
+    this.elementPatterns[name] = selector;
+  }
+
+  /**
+   * Get all supported action types
+   * @returns {string[]}
+   */
+  getSupportedActions() {
+    return Object.keys(this.actionPatterns);
+  }
+}
+
+module.exports = NLParser;