@darbotlabs/darbot-browser-mcp 0.1.1 → 1.3.0

package/lib/ai/integration.js

@@ -0,0 +1,397 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * AI Tool Integration - Executes parsed intents using real browser automation tools
+ */
+export class AIToolIntegration {
+    /**
+     * Execute a parsed intent by calling the appropriate browser automation tool
+     */
+    async executeIntent(context, intent) {
+        try {
+            switch (intent.action) {
+                case 'navigate':
+                    return await this.executeNavigate(context, intent);
+                case 'click':
+                    return await this.executeClick(context, intent);
+                case 'type':
+                    return await this.executeType(context, intent);
+                case 'submit_form':
+                    return await this.executeSubmitForm(context, intent);
+                case 'search':
+                    return await this.executeSearch(context, intent);
+                case 'github_create_issue':
+                    return await this.executeGitHubCreateIssue(context, intent);
+                case 'github_review_pr':
+                    return await this.executeGitHubReviewPR(context, intent);
+                case 'login':
+                    return await this.executeLogin(context, intent);
+                case 'wait_for':
+                    return await this.executeWaitFor(context, intent);
+                case 'screenshot':
+                    return await this.executeScreenshot(context, intent);
+                default:
+                    return await this.executeGeneric(context, intent);
+            }
+        }
+        catch (error) {
+            // Return error result instead of throwing
+            return {
+                success: false,
+                action: intent.action,
+                target: intent.parameters.url || intent.parameters.element,
+                error: error instanceof Error ? error.message : 'Unknown error',
+            };
+        }
+    }
+    /**
+     * Execute recovery strategy when primary intent fails
+     */
+    async executeRecovery(context, intent, primaryError) {
+        const strategy = intent.fallbackStrategy || 'analyze_page_context';
+        try {
+            switch (strategy) {
+                case 'search_for_url':
+                    return await this.recoverSearchForUrl(context, intent);
+                case 'auto_detect_clickable_elements':
+                    return await this.recoverAutoDetectClickable(context, intent);
+                case 'auto_detect_input_fields':
+                    return await this.recoverAutoDetectInput(context, intent);
+                case 'find_submit_buttons':
+                    return await this.recoverFindSubmitButton(context, intent);
+                case 'find_search_input':
+                    return await this.recoverFindSearchInput(context, intent);
+                case 'analyze_page_context':
+                    return await this.recoverAnalyzeContext(context, intent);
+                default:
+                    throw new Error(`Unknown recovery strategy: ${strategy}`);
+            }
+        }
+        catch (recoveryError) {
+            return {
+                success: false,
+                action: intent.action,
+                error: `Primary error: ${primaryError.message}. Recovery failed: ${recoveryError instanceof Error ? recoveryError.message : 'Unknown error'}`,
+                recoveryUsed: true,
+                recoveryStrategy: strategy,
+            };
+        }
+    }
+    // ==================== Action Implementations ====================
+    async executeNavigate(context, intent) {
+        const tab = await context.ensureTab();
+        const url = intent.parameters.url;
+        await tab.navigate(url);
+        return {
+            success: true,
+            action: 'navigate',
+            target: url,
+        };
+    }
+    async executeClick(context, intent) {
+        const tab = context.currentTabOrDie();
+        const element = intent.parameters.element;
+        // Find element by accessibility text or role
+        const locator = await this.findElementByDescription(tab, element);
+        await locator.click();
+        return {
+            success: true,
+            action: 'click',
+            target: element,
+        };
+    }
+    async executeType(context, intent) {
+        const tab = context.currentTabOrDie();
+        const text = intent.parameters.text;
+        const element = intent.parameters.element;
+        // Find input element
+        const locator = await this.findInputByDescription(tab, element);
+        await locator.fill(text);
+        return {
+            success: true,
+            action: 'type',
+            target: element,
+            result: { text },
+        };
+    }
+    async executeSubmitForm(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Find and click submit button, or press Enter on focused element
+        try {
+            const submitButton = tab.page.locator('button[type="submit"], input[type="submit"]').first();
+            await submitButton.click();
+        }
+        catch {
+            // Fallback: press Enter
+            await tab.page.keyboard.press('Enter');
+        }
+        return {
+            success: true,
+            action: 'submit_form',
+        };
+    }
+    async executeSearch(context, intent) {
+        const tab = await context.ensureTab();
+        const query = intent.parameters.query;
+        // Navigate to Google search
+        const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;
+        await tab.navigate(searchUrl);
+        return {
+            success: true,
+            action: 'search',
+            target: query,
+            result: { url: searchUrl },
+        };
+    }
+    async executeGitHubCreateIssue(context, intent) {
+        const tab = context.currentTabOrDie();
+        const currentUrl = tab.page.url();
+        // Extract repository from current URL if on GitHub
+        if (!currentUrl.includes('github.com'))
+            throw new Error('Not on GitHub. Navigate to a repository first.');
+        // Navigate to issues page
+        const repoMatch = currentUrl.match(/github\.com\/([^/]+\/[^/]+)/);
+        if (!repoMatch)
+            throw new Error('Could not determine repository from URL');
+        const issuesUrl = `https://github.com/${repoMatch[1]}/issues/new`;
+        await tab.navigate(issuesUrl);
+        return {
+            success: true,
+            action: 'github_create_issue',
+            target: issuesUrl,
+        };
+    }
+    async executeGitHubReviewPR(context, intent) {
+        const tab = context.currentTabOrDie();
+        const currentUrl = tab.page.url();
+        if (!currentUrl.includes('github.com'))
+            throw new Error('Not on GitHub. Navigate to a repository first.');
+        // Navigate to pull requests
+        const repoMatch = currentUrl.match(/github\.com\/([^/]+\/[^/]+)/);
+        if (!repoMatch)
+            throw new Error('Could not determine repository from URL');
+        const prsUrl = `https://github.com/${repoMatch[1]}/pulls`;
+        await tab.navigate(prsUrl);
+        return {
+            success: true,
+            action: 'github_review_pr',
+            target: prsUrl,
+        };
+    }
+    async executeLogin(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Detect login form on current page
+        const loginInputs = tab.page.locator('input[type="email"], input[type="text"], input[name*="user"], input[name*="email"]');
+        const passwordInputs = tab.page.locator('input[type="password"]');
+        const loginCount = await loginInputs.count();
+        const passwordCount = await passwordInputs.count();
+        if (loginCount === 0 || passwordCount === 0)
+            throw new Error('No login form detected on current page');
+        return {
+            success: true,
+            action: 'login',
+            target: 'login_form_detected',
+            result: {
+                message: 'Login form detected. Use browser_type tool to enter credentials.',
+            },
+        };
+    }
+    async executeWaitFor(context, intent) {
+        const tab = context.currentTabOrDie();
+        const target = intent.parameters.target;
+        // Wait for element or text to appear
+        try {
+            // Try as text content first
+            await tab.page.waitForSelector(`:text("${target}")`, { timeout: 10000 });
+        }
+        catch {
+            // Try as selector
+            await tab.page.waitForSelector(target, { timeout: 10000 });
+        }
+        return {
+            success: true,
+            action: 'wait_for',
+            target,
+        };
+    }
+    async executeScreenshot(context, intent) {
+        const tab = context.currentTabOrDie();
+        const timestamp = Date.now();
+        const filename = `screenshot-${timestamp}.png`;
+        await tab.page.screenshot({ path: filename, fullPage: true });
+        return {
+            success: true,
+            action: 'screenshot',
+            result: { filename, path: filename },
+        };
+    }
+    async executeGeneric(context, intent) {
+        throw new Error(`Action '${intent.action}' is not yet implemented in the integration layer`);
+    }
+    // ==================== Recovery Strategies ====================
+    async recoverSearchForUrl(context, intent) {
+        // If direct navigation failed, try Google search
+        const searchQuery = intent.parameters.url || intent.parameters.element;
+        const searchIntent = {
+            action: 'search',
+            parameters: { query: searchQuery },
+            confidence: 0.6,
+            fallbackStrategy: 'analyze_page_context',
+        };
+        const result = await this.executeSearch(context, searchIntent);
+        result.recoveryUsed = true;
+        result.recoveryStrategy = 'search_for_url';
+        return result;
+    }
+    async recoverAutoDetectClickable(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Find all clickable elements
+        const clickableElements = await tab.page.locator('button, a, [role="button"], [role="link"], input[type="submit"]').all();
+        if (clickableElements.length === 0)
+            throw new Error('No clickable elements found on page');
+        // Click the first visible one
+        await clickableElements[0].click();
+        return {
+            success: true,
+            action: 'click',
+            target: 'auto_detected_element',
+            recoveryUsed: true,
+            recoveryStrategy: 'auto_detect_clickable_elements',
+        };
+    }
+    async recoverAutoDetectInput(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Find visible input fields
+        const inputs = await tab.page.locator('input[type="text"], input[type="email"], input[type="search"], textarea').all();
+        if (inputs.length === 0)
+            throw new Error('No input fields found on page');
+        // Fill the first visible one
+        const text = intent.parameters.text || '';
+        await inputs[0].fill(text);
+        return {
+            success: true,
+            action: 'type',
+            target: 'auto_detected_input',
+            recoveryUsed: true,
+            recoveryStrategy: 'auto_detect_input_fields',
+        };
+    }
+    async recoverFindSubmitButton(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Find submit buttons by various selectors
+        const submitButton = tab.page.locator('button[type="submit"], input[type="submit"], button:has-text("Submit"), button:has-text("Send")').first();
+        await submitButton.click();
+        return {
+            success: true,
+            action: 'submit_form',
+            target: 'auto_detected_submit',
+            recoveryUsed: true,
+            recoveryStrategy: 'find_submit_buttons',
+        };
+    }
+    async recoverFindSearchInput(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Find search input by name, placeholder, or type
+        const searchInput = tab.page.locator('input[type="search"], input[name*="search"], input[placeholder*="search" i]').first();
+        const query = intent.parameters.query || '';
+        await searchInput.fill(query);
+        await searchInput.press('Enter');
+        return {
+            success: true,
+            action: 'search',
+            target: 'auto_detected_search_input',
+            recoveryUsed: true,
+            recoveryStrategy: 'find_search_input',
+        };
+    }
+    async recoverAnalyzeContext(context, intent) {
+        const tab = context.currentTabOrDie();
+        // Capture page snapshot for intelligent recovery
+        const title = await tab.page.title();
+        const url = tab.page.url();
+        // Provide context-aware suggestions
+        const suggestions = [];
+        if (url.includes('github.com'))
+            suggestions.push('Try navigating to issues or pull requests');
+        if (title.toLowerCase().includes('login'))
+            suggestions.push('Page appears to be a login form');
+        throw new Error(`Could not execute action. Suggestions: ${suggestions.join(', ') || 'None'}`);
+    }
+    // ==================== Helper Methods ====================
+    /**
+     * Find element by natural language description
+     */
+    async findElementByDescription(tab, description) {
+        // Try various strategies to find element
+        const strategies = [
+            // 1. Exact text match
+            tab.page.locator(`:text("${description}")`),
+            // 2. Partial text match (case insensitive)
+            tab.page.locator(`:text-is("${description}")`),
+            // 3. Button with text
+            tab.page.locator(`button:has-text("${description}")`),
+            // 4. Link with text
+            tab.page.locator(`a:has-text("${description}")`),
+            // 5. Accessible name
+            tab.page.locator(`[aria-label="${description}"]`),
+            // 6. Placeholder
+            tab.page.locator(`[placeholder="${description}"]`),
+            // 7. Title attribute
+            tab.page.locator(`[title="${description}"]`),
+        ];
+        // Try each strategy
+        for (const locator of strategies) {
+            const count = await locator.count();
+            if (count > 0)
+                return locator.first();
+        }
+        // If all strategies fail, throw error
+        throw new Error(`Could not find element matching: ${description}`);
+    }
+    /**
+     * Find input field by natural language description
+     */
+    async findInputByDescription(tab, description) {
+        // Try various input-specific strategies
+        const strategies = [
+            // 1. Label text
+            tab.page.locator(`input:near(:text("${description}"))`),
+            // 2. Placeholder
+            tab.page.locator(`input[placeholder*="${description}" i]`),
+            // 3. Name attribute
+            tab.page.locator(`input[name*="${description}" i]`),
+            // 4. Accessible label
+            tab.page.locator(`input[aria-label*="${description}" i]`),
+            // 5. ID
+            tab.page.locator(`input[id*="${description}" i]`),
+            // 6. Any input (fallback)
+            tab.page.locator('input[type="text"], input[type="email"], input[type="search"], textarea').first(),
+        ];
+        for (const locator of strategies) {
+            try {
+                const count = await locator.count();
+                if (count > 0)
+                    return locator.first();
+            }
+            catch {
+                continue;
+            }
+        }
+        throw new Error(`Could not find input field matching: ${description}`);
+    }
+}
+// Global integration instance
+export const aiToolIntegration = new AIToolIntegration();

package/lib/ai/intent.js

@@ -0,0 +1,237 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Intent parser for natural language automation commands
+ */
+export class IntentParser {
+    patterns = [
+        // Navigation intents
+        {
+            pattern: /(?:go to|navigate to|visit|open)\s+(.+)/i,
+            action: 'navigate',
+            parameterExtractors: {
+                url: match => this.normalizeUrl(match[1].trim()),
+            },
+            confidence: 0.9,
+        },
+        // Form submission intents
+        {
+            pattern: /(?:submit|send|complete)\s+(?:the\s+)?form/i,
+            action: 'submit_form',
+            parameterExtractors: {},
+            confidence: 0.85,
+        },
+        // Click intents
+        {
+            pattern: /(?:click|press|tap)\s+(?:on\s+)?(?:the\s+)?(.+?)(?:\s+button|\s+link)?$/i,
+            action: 'click',
+            parameterExtractors: {
+                element: match => match[1].trim(),
+            },
+            confidence: 0.8,
+        },
+        // Text input intents
+        {
+            pattern: /(?:type|enter|input|fill)\s+['""](.+?)['""]\s+(?:in|into)\s+(?:the\s+)?(.+)/i,
+            action: 'type',
+            parameterExtractors: {
+                text: match => match[1],
+                element: match => match[2].trim(),
+            },
+            confidence: 0.85,
+        },
+        // Search intents
+        {
+            pattern: /search\s+for\s+['""]?(.+?)['""]?/i,
+            action: 'search',
+            parameterExtractors: {
+                query: match => match[1].trim(),
+            },
+            confidence: 0.8,
+        },
+        // GitHub-specific intents
+        {
+            pattern: /(?:create|open)\s+(?:a\s+)?(?:new\s+)?(?:github\s+)?issue/i,
+            action: 'github_create_issue',
+            parameterExtractors: {},
+            confidence: 0.9,
+        },
+        {
+            pattern: /(?:review|check)\s+(?:the\s+)?(?:github\s+)?(?:pull\s+)?(?:request|pr)/i,
+            action: 'github_review_pr',
+            parameterExtractors: {},
+            confidence: 0.9,
+        },
+        // Login intents
+        {
+            pattern: /(?:log\s*in|sign\s*in|login)\s+(?:to\s+)?(.+)/i,
+            action: 'login',
+            parameterExtractors: {
+                service: match => match[1].trim(),
+            },
+            confidence: 0.85,
+        },
+        // Wait intents
+        {
+            pattern: /wait\s+(?:for\s+)?(?:the\s+)?(.+?)(?:\s+to\s+(?:appear|load|show))?/i,
+            action: 'wait_for',
+            parameterExtractors: {
+                target: match => match[1].trim(),
+            },
+            confidence: 0.8,
+        },
+        // Screenshot intents
+        {
+            pattern: /(?:take|capture)\s+(?:a\s+)?screenshot/i,
+            action: 'screenshot',
+            parameterExtractors: {},
+            confidence: 0.9,
+        },
+    ];
+    /**
+     * Parse natural language intent into structured action
+     */
+    parseIntent(description) {
+        const normalizedDescription = description.trim();
+        for (const pattern of this.patterns) {
+            const match = normalizedDescription.match(pattern.pattern);
+            if (match) {
+                const parameters = {};
+                // Extract parameters using pattern extractors
+                for (const [key, extractor] of Object.entries(pattern.parameterExtractors)) {
+                    try {
+                        parameters[key] = extractor(match);
+                    }
+                    catch (error) {
+                        // Skip parameter if extraction fails
+                        continue;
+                    }
+                }
+                return {
+                    action: pattern.action,
+                    parameters,
+                    confidence: pattern.confidence,
+                    fallbackStrategy: this.getFallbackStrategy(pattern.action),
+                };
+            }
+        }
+        // Fallback to generic action detection
+        return this.parseGenericIntent(normalizedDescription);
+    }
+    /**
+     * Parse generic intents that don't match specific patterns
+     */
+    parseGenericIntent(description) {
+        const lowercaseDesc = description.toLowerCase();
+        // Common action keywords
+        if (lowercaseDesc.includes('click') || lowercaseDesc.includes('press')) {
+            return {
+                action: 'click',
+                parameters: { element: description },
+                confidence: 0.6,
+                fallbackStrategy: 'auto_detect_clickable_elements',
+            };
+        }
+        if (lowercaseDesc.includes('type') || lowercaseDesc.includes('enter')) {
+            return {
+                action: 'type',
+                parameters: { text: description },
+                confidence: 0.5,
+                fallbackStrategy: 'auto_detect_input_fields',
+            };
+        }
+        if (lowercaseDesc.includes('navigate') || lowercaseDesc.includes('go')) {
+            return {
+                action: 'navigate',
+                parameters: { url: description },
+                confidence: 0.5,
+                fallbackStrategy: 'search_for_url',
+            };
+        }
+        // Default to generic interaction
+        return {
+            action: 'interact',
+            parameters: { description },
+            confidence: 0.3,
+            fallbackStrategy: 'analyze_page_context',
+        };
+    }
+    /**
+     * Get fallback strategy for action type
+     */
+    getFallbackStrategy(action) {
+        const strategies = {
+            'navigate': 'search_for_url',
+            'click': 'auto_detect_clickable_elements',
+            'type': 'auto_detect_input_fields',
+            'submit_form': 'find_submit_buttons',
+            'search': 'find_search_input',
+            'github_create_issue': 'navigate_to_github_issues',
+            'github_review_pr': 'navigate_to_github_prs',
+            'login': 'find_login_form',
+            'wait_for': 'intelligent_wait',
+            'screenshot': 'capture_full_page',
+        };
+        return strategies[action] || 'analyze_page_context';
+    }
+    /**
+     * Normalize URL for navigation
+     */
+    normalizeUrl(url) {
+        // Remove quotes and trim
+        const cleaned = url.replace(/['"]/g, '').trim();
+        // Add protocol if missing
+        if (!cleaned.startsWith('http://') && !cleaned.startsWith('https://')) {
+            // Check if it looks like a domain
+            if (cleaned.includes('.') && !cleaned.includes(' '))
+                return `https://${cleaned}`;
+            // Otherwise treat as search query
+            return `https://www.google.com/search?q=${encodeURIComponent(cleaned)}`;
+        }
+        return cleaned;
+    }
+    /**
+     * Enhance intent with context information
+     */
+    enhanceWithContext(intent, context) {
+        // Add context-aware enhancements
+        const enhanced = { ...intent };
+        // Enhance based on current page context
+        if (context.pageIntent)
+            enhanced.parameters.pageContext = context.pageIntent;
+        // Adjust confidence based on context
+        if (context.successfulActions) {
+            const similarActions = context.successfulActions.filter((action) => action.action === intent.action);
+            if (similarActions.length > 0)
+                enhanced.confidence = Math.min(enhanced.confidence + 0.1, 1.0);
+        }
+        return enhanced;
+    }
+    /**
+     * Extract workflow intentions from complex descriptions
+     */
+    parseWorkflowIntent(description) {
+        // Split complex descriptions into steps
+        const stepSeparators = /(?:then|next|after that|and then|,\s*)/i;
+        const steps = description.split(stepSeparators).map(s => s.trim()).filter(s => s.length > 0);
+        if (steps.length <= 1)
+            return [this.parseIntent(description)];
+        // Parse each step as individual intent
+        return steps.map(step => this.parseIntent(step));
+    }
+}
+// Global intent parser instance
+export const intentParser = new IntentParser();