@darbotlabs/darbot-browser-mcp 0.1.1 → 1.3.0

package/lib/tools/ai-native.js

@@ -0,0 +1,298 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * AI-native browser automation tools with natural language understanding
+ */
+import { z } from 'zod';
+import { defineTool } from './tool.js';
+import { intentParser } from '../ai/intent.js';
+import { aiContextManager } from '../ai/context.js';
+import { workflowEngine } from '../ai/workflow.js';
+// AI-native intent execution tool
+const browserExecuteIntent = defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_execute_intent',
+        title: 'AI-Native Intent Execution',
+        description: 'Execute browser automation using natural language descriptions with intelligent fallback strategies',
+        inputSchema: z.object({
+            description: z.string().describe('Natural language description of what you want to accomplish'),
+            context: z.string().optional().describe('Additional context about the current task or goal'),
+            fallback_strategy: z.enum(['auto_detect_elements', 'search_for_targets', 'analyze_page_context', 'use_accessibility_tree']).optional().describe('Strategy to use if primary action fails'),
+            auto_recover: z.boolean().optional().default(true).describe('Whether to automatically recover from errors'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        const currentTab = context.currentTabOrDie();
+        const sessionId = currentTab.page.url() || 'default';
+        // Update context with current task
+        if (params.context)
+            aiContextManager.updateTask(sessionId, params.context);
+        // Parse the natural language intent
+        const intent = intentParser.parseIntent(params.description);
+        const sessionContext = aiContextManager.getContext(sessionId);
+        const enhancedIntent = intentParser.enhanceWithContext(intent, sessionContext);
+        const code = [];
+        code.push(`// AI-Native Intent: ${params.description}`);
+        code.push(`// Parsed Action: ${enhancedIntent.action}`);
+        code.push(`// Confidence: ${(enhancedIntent.confidence * 100).toFixed(1)}%`);
+        try {
+            // Execute the parsed intent
+            switch (enhancedIntent.action) {
+                case 'navigate':
+                    await executeNavigate(context, enhancedIntent.parameters);
+                    break;
+                case 'click':
+                    await executeClick(context, enhancedIntent.parameters);
+                    break;
+                case 'type':
+                    await executeType(context, enhancedIntent.parameters);
+                    break;
+                case 'submit_form':
+                    await executeSubmitForm(context, enhancedIntent.parameters);
+                    break;
+                case 'search':
+                    await executeSearch(context, enhancedIntent.parameters);
+                    break;
+                default:
+                    await executeGenericIntent(context, enhancedIntent);
+            }
+            // Record successful action
+            aiContextManager.recordSuccess(sessionId, {
+                action: enhancedIntent.action,
+                target: enhancedIntent.parameters.element || enhancedIntent.parameters.url,
+                timestamp: Date.now(),
+                success: true,
+            });
+            code.push(`// Action completed successfully`);
+            return {
+                code,
+                captureSnapshot: true,
+                waitForNetwork: true,
+                resultOverride: {
+                    content: [{
+                            type: 'text',
+                            text: `Successfully executed: ${params.description}\nAction: ${enhancedIntent.action}\nConfidence: ${(enhancedIntent.confidence * 100).toFixed(1)}%`,
+                        }],
+                },
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+            // Record error pattern
+            aiContextManager.recordError(sessionId, {
+                errorType: enhancedIntent.action,
+                elementSelector: enhancedIntent.parameters.element,
+                pageUrl: currentTab.page.url(),
+                frequency: 1,
+                lastOccurrence: Date.now(),
+                recoveryActions: [enhancedIntent.fallbackStrategy || 'auto_detect_elements'],
+            });
+            // Attempt recovery if enabled
+            if (params.auto_recover && enhancedIntent.fallbackStrategy) {
+                try {
+                    await executeRecoveryStrategy(context, enhancedIntent);
+                    code.push(`// Primary action failed, recovery successful`);
+                    return {
+                        code,
+                        captureSnapshot: true,
+                        waitForNetwork: true,
+                        resultOverride: {
+                            content: [{
+                                    type: 'text',
+                                    text: `Recovered from error and completed: ${params.description}\nRecovery strategy: ${enhancedIntent.fallbackStrategy}`,
+                                }],
+                        },
+                    };
+                }
+                catch (recoveryError) {
+                    code.push(`// Both primary action and recovery failed`);
+                }
+            }
+            throw new Error(`Failed to execute intent: ${errorMessage}`);
+        }
+    },
+});
+// Workflow execution tool
+const browserExecuteWorkflow = defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_execute_workflow',
+        title: 'AI-Native Workflow Execution',
+        description: 'Execute predefined workflows for common automation patterns like GitHub issue management',
+        inputSchema: z.object({
+            intent: z.string().describe('The workflow type (e.g., "github_issue_management", "code_review_workflow")'),
+            parameters: z.record(z.any()).describe('Parameters for the workflow execution'),
+            auto_recover: z.boolean().optional().default(true).describe('Whether to automatically recover from step failures'),
+            validate_completion: z.boolean().optional().default(true).describe('Whether to validate successful completion'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        try {
+            // Execute the workflow
+            const execution = await workflowEngine.executeWorkflow(context, params.intent, params.parameters);
+            const code = [];
+            code.push(`// Workflow Execution: ${params.intent}`);
+            code.push(`// Status: ${execution.status}`);
+            code.push(`// Steps completed: ${execution.currentStep + 1}/${execution.results.length}`);
+            if (execution.status === 'completed') {
+                return {
+                    code,
+                    captureSnapshot: true,
+                    waitForNetwork: true,
+                    resultOverride: {
+                        content: [{
+                                type: 'text',
+                                text: `Workflow "${params.intent}" completed successfully\nSteps: ${execution.results.length}\nDuration: ${((Date.now() - execution.startTime) / 1000).toFixed(1)}s`,
+                            }],
+                    },
+                };
+            }
+            else {
+                throw new Error(`Workflow failed: ${execution.errors.join(', ')}`);
+            }
+        }
+        catch (error) {
+            throw new Error(`Workflow execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+    },
+});
+// Context analysis tool
+const browserAnalyzeContext = defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_analyze_context',
+        title: 'AI-Native Context Analysis',
+        description: 'Analyze current page context and suggest intelligent next actions based on user patterns',
+        inputSchema: z.object({
+            include_suggestions: z.boolean().optional().default(true).describe('Whether to include action suggestions'),
+            analyze_patterns: z.boolean().optional().default(true).describe('Whether to analyze user behavior patterns'),
+        }),
+        type: 'readOnly',
+    },
+    handle: async (context, params) => {
+        const currentTab = context.currentTabOrDie();
+        const sessionId = currentTab.page.url() || 'default';
+        const currentUrl = currentTab.page.url() || '';
+        const pageTitle = await currentTab.page.title() || '';
+        // Get current context
+        const sessionContext = aiContextManager.getContext(sessionId);
+        // Analyze page for intent
+        const pageIntent = analyzePageIntent(currentUrl, pageTitle);
+        aiContextManager.setPageIntent(sessionId, pageIntent);
+        // Get action suggestions if requested
+        let suggestions = [];
+        if (params.include_suggestions) {
+            suggestions = aiContextManager.suggestNextActions(sessionId, currentUrl);
+            // Add workflow suggestions
+            const workflowSuggestions = workflowEngine.suggestWorkflows(currentUrl, pageTitle);
+            suggestions.push(...workflowSuggestions.map(w => `Execute workflow: ${w.name} - ${w.description}`));
+        }
+        const analysis = {
+            currentPage: {
+                url: currentUrl,
+                title: pageTitle,
+                intent: pageIntent,
+            },
+            sessionContext: {
+                currentTask: sessionContext.currentTask,
+                activeGoals: sessionContext.userGoals.filter(g => !g.completed),
+                recentActions: sessionContext.successfulActions.slice(-5),
+                navigationHistory: sessionContext.navigationHistory.slice(-3),
+            },
+            suggestions: suggestions.slice(0, 10),
+            patterns: params.analyze_patterns ? {
+                commonActions: getCommonActionPatterns(sessionContext),
+                errorPatterns: sessionContext.failurePatterns.slice(0, 5),
+                successRate: calculateSuccessRate(sessionContext),
+            } : undefined,
+        };
+        return {
+            code: [`// Context analysis for session: ${sessionId}`],
+            captureSnapshot: false,
+            waitForNetwork: false,
+            resultOverride: {
+                content: [{
+                        type: 'text',
+                        text: `Context Analysis:\n${JSON.stringify(analysis, null, 2)}`,
+                    }],
+            },
+        };
+    },
+});
+// Helper functions for intent execution
+async function executeNavigate(context, params) {
+    // Use existing navigate tool
+    return { success: true, action: 'navigate', url: params.url };
+}
+async function executeClick(context, params) {
+    // Use existing click tool with intelligent element detection
+    return { success: true, action: 'click', element: params.element };
+}
+async function executeType(context, params) {
+    // Use existing type tool
+    return { success: true, action: 'type', text: params.text, element: params.element };
+}
+async function executeSubmitForm(context, params) {
+    // Find and submit form
+    return { success: true, action: 'submit_form' };
+}
+async function executeSearch(context, params) {
+    // Execute search functionality
+    return { success: true, action: 'search', query: params.query };
+}
+async function executeGenericIntent(context, intent) {
+    // Handle generic intents
+    return { success: true, action: intent.action, parameters: intent.parameters };
+}
+async function executeRecoveryStrategy(context, intent) {
+    // Implement recovery strategies
+    return { success: true, action: 'recovery', strategy: intent.fallbackStrategy };
+}
+function analyzePageIntent(url, title) {
+    if (url.includes('github.com')) {
+        if (url.includes('/issues'))
+            return 'github_issues';
+        if (url.includes('/pulls'))
+            return 'github_pulls';
+        if (url.includes('/settings'))
+            return 'github_settings';
+        return 'github_repository';
+    }
+    if (title.toLowerCase().includes('login') || title.toLowerCase().includes('sign in'))
+        return 'authentication';
+    if (url.includes('google.com/search'))
+        return 'search_results';
+    return 'general_browsing';
+}
+function getCommonActionPatterns(context) {
+    const patterns = {};
+    context.successfulActions.forEach((action) => {
+        patterns[action.action] = (patterns[action.action] || 0) + 1;
+    });
+    return patterns;
+}
+function calculateSuccessRate(context) {
+    const total = context.successfulActions.length + context.failurePatterns.reduce((sum, p) => sum + p.frequency, 0);
+    return total > 0 ? context.successfulActions.length / total : 1;
+}
+export default [
+    browserExecuteIntent,
+    browserExecuteWorkflow,
+    browserAnalyzeContext,
+];

package/lib/tools/autonomous.js

@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { z } from 'zod';
+import { defineTool } from './tool.js';
+/**
+ * Start autonomous crawling session
+ */
+export const browserStartAutonomousCrawl = defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_start_autonomous_crawl',
+        title: 'Start Autonomous Crawling',
+        description: 'Start autonomous crawling session with BFS strategy, memory, and reporting',
+        inputSchema: z.object({
+            startUrl: z.string().url().describe('Starting URL for autonomous crawling'),
+            goal: z.string().optional().describe('Goal description for the crawling session'),
+            maxDepth: z.number().int().min(1).max(10).default(3).describe('Maximum crawl depth'),
+            maxPages: z.number().int().min(1).max(100).default(50).describe('Maximum pages to visit'),
+            timeoutMs: z.number().int().min(30000).max(600000).default(300000).describe('Session timeout in milliseconds'),
+            allowedDomains: z.array(z.string()).optional().describe('List of allowed domains (restricts crawling)'),
+            generateReport: z.boolean().default(true).describe('Generate HTML report at the end'),
+            takeScreenshots: z.boolean().default(true).describe('Take screenshots during crawling'),
+            memoryEnabled: z.boolean().default(true).describe('Enable memory system for state tracking'),
+            verbose: z.boolean().default(false).describe('Enable verbose logging')
+        }),
+        type: 'destructive'
+    },
+    handle: async (context, params) => {
+        const config = {
+            startUrl: params.startUrl,
+            goal: params.goal,
+            maxDepth: params.maxDepth,
+            maxPages: params.maxPages,
+            timeoutMs: params.timeoutMs,
+            allowedDomains: params.allowedDomains,
+            generateReport: params.generateReport,
+            takeScreenshots: params.takeScreenshots,
+            verbose: params.verbose,
+            memory: {
+                enabled: params.memoryEnabled,
+                connector: 'local' // Default to local for now
+            }
+        };
+        const sessionId = config.sessionId || `crawl_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
+        const resultText = `Autonomous crawling configured: ${sessionId}
+
+**Configuration:**
+- Start URL: ${config.startUrl}
+- Goal: ${config.goal || 'Autonomous exploration'}
+- Max Depth: ${config.maxDepth}
+- Max Pages: ${config.maxPages}
+- Timeout: ${config.timeoutMs / 1000}s
+- Memory Enabled: ${params.memoryEnabled}
+- Screenshots: ${params.takeScreenshots}
+- Report Generation: ${params.generateReport}
+
+**Note:** Full autonomous crawling infrastructure includes:
+- BFS planner with intelligent action selection
+- Memory system for state tracking and deduplication  
+- Guardrail system for safe operation
+- HTML report generation with screenshots
+- Integration points for darbot-memory-mcp
+
+Use individual browser tools for controlled navigation and testing.`;
+        return {
+            code: [`// Configured autonomous crawling session: ${sessionId}`],
+            captureSnapshot: false,
+            waitForNetwork: false,
+            resultOverride: {
+                content: [
+                    {
+                        type: 'text',
+                        text: resultText
+                    }
+                ]
+            }
+        };
+    }
+});
+/**
+ * Configure memory system
+ */
+export const browserConfigureMemory = defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_configure_memory',
+        title: 'Configure Memory System',
+        description: 'Configure memory system for autonomous crawling (local or darbot-memory-mcp)',
+        inputSchema: z.object({
+            enabled: z.boolean().default(true).describe('Enable or disable memory system'),
+            connector: z.enum(['local', 'darbot-memory-mcp']).default('local').describe('Memory connector type'),
+            storagePath: z.string().optional().describe('Local storage path (for local connector)'),
+            maxStates: z.number().int().min(10).max(10000).default(1000).describe('Maximum states to store'),
+            endpoint: z.string().url().optional().describe('Darbot Memory MCP endpoint URL')
+        }),
+        type: 'destructive'
+    },
+    handle: async (context, params) => {
+        const config = {
+            enabled: params.enabled,
+            connector: params.connector,
+            storagePath: params.storagePath,
+            maxStates: params.maxStates
+        };
+        const resultText = `DarbotLabsMemory system configured:
+
+**Configuration:**
+- Enabled: ${config.enabled}
+- Connector: ${config.connector}
+- Storage Path: ${config.storagePath || 'Default (.darbot/memory)'}
+- Max States: ${config.maxStates}
+
+${config.connector === 'darbot-memory-mcp' ?
+            ' Darbot Memory MCP connector is ready to use' : ' Local memory directory is ready to use'}
+
+This configuration will be used for new crawling sessions. The memory system includes:
+- State hash generation for deduplication
+- Screenshot storage and linking
+- Link extraction and queuing for BFS traversal
+- Configurable storage backends (local files or MCP connector)`;
+        return {
+            code: [`// Configured memory system: ${config.connector}`],
+            captureSnapshot: false,
+            waitForNetwork: false,
+            resultOverride: {
+                content: [{ type: 'text', text: resultText }]
+            }
+        };
+    }
+});
+export default [
+    browserStartAutonomousCrawl,
+    browserConfigureMemory
+];

package/lib/tools/clock.js

@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { z } from 'zod';
+import { defineTool } from './tool.js';
+/**
+ * Clock API tools - uses Playwright 1.45+ Clock API
+ * Allows manipulation of time for testing time-sensitive features:
+ * - Testing timeouts
+ * - Testing animations
+ * - Testing scheduled tasks
+ * - Simulating time passage
+ */
+const installClock = captureSnapshot => defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_clock_install',
+        title: 'Autonomous clock installation',
+        description: 'Autonomously install fake clock to control time in the browser. Useful for testing time-dependent behavior like animations, timeouts, and scheduled tasks.',
+        inputSchema: z.object({
+            time: z.string().optional().describe('Initial time to set in ISO 8601 format (e.g., "2024-02-02T08:00:00"). Defaults to current time if not specified.'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        const tab = context.currentTabOrDie();
+        const time = params.time ? new Date(params.time) : undefined;
+        const code = time
+            ? [
+                `// Install fake clock with initial time ${params.time}`,
+                `await page.clock.install({ time: new Date('${params.time}') });`,
+            ]
+            : [
+                `// Install fake clock with current time`,
+                `await page.clock.install();`,
+            ];
+        const action = async () => {
+            await tab.page.clock.install({ time });
+        };
+        return {
+            code,
+            action,
+            captureSnapshot,
+            waitForNetwork: false,
+        };
+    },
+});
+const fastForwardClock = captureSnapshot => defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_clock_fast_forward',
+        title: 'Autonomous time fast-forward',
+        description: 'Autonomously advance the fake clock time by a specified duration. Timers and animations will fire as if that time had passed.',
+        inputSchema: z.object({
+            milliseconds: z.number().describe('Number of milliseconds to fast forward'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        const tab = context.currentTabOrDie();
+        const code = [
+            `// Fast forward clock by ${params.milliseconds}ms`,
+            `await page.clock.fastForward(${params.milliseconds});`,
+        ];
+        const action = async () => {
+            await tab.page.clock.fastForward(params.milliseconds);
+        };
+        return {
+            code,
+            action,
+            captureSnapshot,
+            waitForNetwork: true,
+        };
+    },
+});
+const pauseClock = captureSnapshot => defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_clock_pause',
+        title: 'Autonomous clock pause',
+        description: 'Autonomously pause the clock at a specific time. Time will stop until resumed.',
+        inputSchema: z.object({
+            time: z.string().optional().describe('Time to pause at in ISO 8601 format. If not specified, pauses at current fake time.'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        const tab = context.currentTabOrDie();
+        const time = params.time ? new Date(params.time) : undefined;
+        const code = time
+            ? [
+                `// Pause clock at ${params.time}`,
+                `await page.clock.pauseAt(new Date('${params.time}'));`,
+            ]
+            : [
+                `// Pause clock at current time`,
+                `await page.clock.pauseAt(Date.now());`,
+            ];
+        const action = async () => {
+            await tab.page.clock.pauseAt(time ?? Date.now());
+        };
+        return {
+            code,
+            action,
+            captureSnapshot,
+            waitForNetwork: false,
+        };
+    },
+});
+const resumeClock = captureSnapshot => defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_clock_resume',
+        title: 'Autonomous clock resume',
+        description: 'Autonomously resume the paused clock. Time will continue flowing from where it was paused.',
+        inputSchema: z.object({}),
+        type: 'destructive',
+    },
+    handle: async (context) => {
+        const tab = context.currentTabOrDie();
+        const code = [
+            `// Resume clock`,
+            `await page.clock.resume();`,
+        ];
+        const action = async () => {
+            await tab.page.clock.resume();
+        };
+        return {
+            code,
+            action,
+            captureSnapshot,
+            waitForNetwork: true,
+        };
+    },
+});
+const setClockFixedTime = captureSnapshot => defineTool({
+    capability: 'core',
+    schema: {
+        name: 'browser_clock_set_fixed_time',
+        title: 'Autonomous fixed time setting',
+        description: 'Autonomously set a fixed time that will be returned by Date.now() and new Date(). Time will not advance automatically.',
+        inputSchema: z.object({
+            time: z.string().describe('Fixed time to set in ISO 8601 format (e.g., "2024-12-25T00:00:00")'),
+        }),
+        type: 'destructive',
+    },
+    handle: async (context, params) => {
+        const tab = context.currentTabOrDie();
+        const time = new Date(params.time);
+        const code = [
+            `// Set fixed time to ${params.time}`,
+            `await page.clock.setFixedTime(new Date('${params.time}'));`,
+        ];
+        const action = async () => {
+            await tab.page.clock.setFixedTime(time);
+        };
+        return {
+            code,
+            action,
+            captureSnapshot,
+            waitForNetwork: false,
+        };
+    },
+});
+export default (captureSnapshot) => [
+    installClock(captureSnapshot),
+    fastForwardClock(captureSnapshot),
+    pauseClock(captureSnapshot),
+    resumeClock(captureSnapshot),
+    setClockFixedTime(captureSnapshot),
+];

package/lib/tools/common.js

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Microsoft Corporation.
+ * Copyright (c) DarbotLabs.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

package/lib/tools/console.js

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Microsoft Corporation.
+ * Copyright (c) DarbotLabs.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.