@darbotlabs/darbot-browser-mcp 0.2.0 → 1.3.0

package/lib/planner.js

@@ -0,0 +1,302 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import debug from 'debug';
+const log = debug('darbot:planner');
+/**
+ * BFS Planner for autonomous site crawling
+ */
+export class BFSPlanner {
+    config;
+    memory;
+    visitQueue;
+    visited;
+    currentDepth = 0;
+    pagesVisited = 0;
+    constructor(config, memory) {
+        this.config = {
+            ...config,
+            // Set default strategy if not provided
+            strategy: config.strategy || 'bfs'
+        };
+        this.memory = memory;
+        this.visitQueue = [];
+        this.visited = new Set();
+    }
+    /**
+     * Plan the next action based on current observation and goal
+     */
+    async planNextAction(observation) {
+        try {
+            log('Planning next action for:', observation.url);
+            // Check if we should finish crawling
+            if (this.shouldFinish(observation)) {
+                return {
+                    type: 'finish',
+                    reason: this.getFinishReason(),
+                    priority: 1
+                };
+            }
+            // Mark current URL as visited
+            this.visited.add(observation.url);
+            this.pagesVisited++;
+            // Store current state in memory
+            if (this.memory.enabled) {
+                await this.memory.storeState(observation.url, observation.title, observation.domSnapshot, undefined, // Screenshots handled separately
+                observation.links.map(link => link.href));
+            }
+            // Extract and queue new links
+            await this.extractAndQueueLinks(observation);
+            // Get next URL to visit
+            const nextTarget = this.getNextTarget();
+            if (nextTarget) {
+                return {
+                    type: 'navigate',
+                    url: nextTarget.url,
+                    reason: `BFS navigation to depth ${nextTarget.depth}: ${nextTarget.url}`,
+                    priority: this.calculatePriority(nextTarget.url)
+                };
+            }
+            // If no more URLs to visit, check for clickable elements on current page
+            const clickTarget = this.findBestClickTarget(observation);
+            if (clickTarget) {
+                return {
+                    type: 'click',
+                    target: clickTarget.selector,
+                    reason: `Clicking "${clickTarget.text}" to discover new content`,
+                    priority: 2
+                };
+            }
+            // No more actions possible
+            return {
+                type: 'finish',
+                reason: 'No more discoverable content within constraints',
+                priority: 1
+            };
+        }
+        catch (error) {
+            log('Error in planning:', error);
+            return {
+                type: 'finish',
+                reason: `Planning error: ${error}`,
+                priority: 1
+            };
+        }
+    }
+    /**
+     * Initialize the planner with a starting URL
+     */
+    async initialize(startUrl) {
+        this.visitQueue.push({ url: startUrl, depth: 0 });
+        log('Initialized BFS planner with start URL:', startUrl);
+    }
+    /**
+     * Extract links from observation and add to queue
+     */
+    async extractAndQueueLinks(observation) {
+        const currentDepth = this.getCurrentDepth(observation.url);
+        for (const link of observation.links) {
+            const normalizedUrl = this.normalizeUrl(link.href, observation.url);
+            if (this.shouldVisitUrl(normalizedUrl, currentDepth + 1)) {
+                if (!this.visited.has(normalizedUrl) && !this.isQueued(normalizedUrl)) {
+                    this.visitQueue.push({
+                        url: normalizedUrl,
+                        depth: currentDepth + 1,
+                        parent: observation.url
+                    });
+                    log('Queued URL:', normalizedUrl, 'at depth', currentDepth + 1);
+                }
+            }
+        }
+        // Sort queue by priority (breadth-first)
+        this.visitQueue.sort((a, b) => {
+            if (a.depth !== b.depth)
+                return a.depth - b.depth;
+            return this.calculatePriority(b.url) - this.calculatePriority(a.url);
+        });
+    }
+    /**
+     * Get the next target URL from the queue
+     */
+    getNextTarget() {
+        while (this.visitQueue.length > 0) {
+            const target = this.visitQueue.shift();
+            if (!this.visited.has(target.url) && this.shouldVisitUrl(target.url, target.depth))
+                return target;
+        }
+        return null;
+    }
+    /**
+     * Find the best clickable element to interact with
+     */
+    findBestClickTarget(observation) {
+        const clickableElements = observation.clickableElements
+            .filter(el => this.isInterestingElement(el))
+            .sort((a, b) => this.calculateElementPriority(b) - this.calculateElementPriority(a));
+        return clickableElements.length > 0 ? clickableElements[0] : null;
+    }
+    /**
+     * Check if an element is interesting to click
+     */
+    isInterestingElement(element) {
+        const text = element.text.toLowerCase();
+        const tag = element.tag.toLowerCase();
+        // Avoid navigation elements we might get stuck in
+        const avoidPatterns = [
+            /back/i, /previous/i, /close/i, /cancel/i, /logout/i, /sign.?out/i,
+            /advertisement/i, /ad/i, /sponsor/i, /cookie/i, /privacy/i
+        ];
+        if (avoidPatterns.some(pattern => pattern.test(text)))
+            return false;
+        // Prefer buttons and links with meaningful text
+        if (['button', 'a', 'input'].includes(tag) && text.length > 2)
+            return true;
+        return false;
+    }
+    /**
+     * Calculate priority for clicking an element
+     */
+    calculateElementPriority(element) {
+        let priority = 0;
+        const text = element.text.toLowerCase();
+        // Higher priority for content-related actions
+        const highPriorityPatterns = [
+            /more/i, /show/i, /view/i, /read/i, /details/i, /expand/i,
+            /next/i, /continue/i, /load/i, /see.?all/i
+        ];
+        if (highPriorityPatterns.some(pattern => pattern.test(text)))
+            priority += 10;
+        // Medium priority for navigation
+        const mediumPriorityPatterns = [
+            /menu/i, /navigation/i, /search/i, /filter/i, /category/i
+        ];
+        if (mediumPriorityPatterns.some(pattern => pattern.test(text)))
+            priority += 5;
+        return priority;
+    }
+    /**
+     * Calculate priority for visiting a URL
+     */
+    calculatePriority(url) {
+        let priority = 0;
+        // Higher priority for pages with interesting content indicators
+        const highPriorityPatterns = [
+            /article/i, /post/i, /blog/i, /news/i, /content/i,
+            /product/i, /item/i, /detail/i, /page/i
+        ];
+        const mediumPriorityPatterns = [
+            /category/i, /section/i, /topic/i, /tag/i, /archive/i
+        ];
+        const lowPriorityPatterns = [
+            /admin/i, /login/i, /register/i, /account/i, /profile/i,
+            /terms/i, /privacy/i, /legal/i, /contact/i, /about/i
+        ];
+        if (highPriorityPatterns.some(pattern => pattern.test(url)))
+            priority += 10;
+        else if (mediumPriorityPatterns.some(pattern => pattern.test(url)))
+            priority += 5;
+        else if (lowPriorityPatterns.some(pattern => pattern.test(url)))
+            priority -= 5;
+        return priority;
+    }
+    /**
+     * Check if we should visit a URL
+     */
+    shouldVisitUrl(url, depth) {
+        if (depth > this.config.maxDepth)
+            return false;
+        if (this.pagesVisited >= this.config.maxPages)
+            return false;
+        // Check allowed domains
+        if (this.config.allowedDomains && this.config.allowedDomains.length > 0) {
+            try {
+                const urlObj = new URL(url);
+                const allowed = this.config.allowedDomains.some(domain => urlObj.hostname === domain || urlObj.hostname.endsWith('.' + domain));
+                if (!allowed)
+                    return false;
+            }
+            catch {
+                return false;
+            }
+        }
+        // Check exclude patterns
+        if (this.config.excludePatterns) {
+            if (this.config.excludePatterns.some(pattern => pattern.test(url)))
+                return false;
+        }
+        // Exclude common file types and non-HTML resources
+        const fileExtensionPattern = /\.(pdf|doc|docx|xls|xlsx|ppt|pptx|zip|rar|exe|dmg|mp4|mp3|jpg|jpeg|png|gif|svg|css|js|json|xml)$/i;
+        if (fileExtensionPattern.test(url))
+            return false;
+        return true;
+    }
+    /**
+     * Check if URL is already queued
+     */
+    isQueued(url) {
+        return this.visitQueue.some(item => item.url === url);
+    }
+    /**
+     * Get current depth for a URL
+     */
+    getCurrentDepth(url) {
+        const queueItem = this.visitQueue.find(item => item.url === url);
+        return queueItem ? queueItem.depth : this.currentDepth;
+    }
+    /**
+     * Normalize URL relative to base URL
+     */
+    normalizeUrl(href, baseUrl) {
+        try {
+            return new URL(href, baseUrl).href;
+        }
+        catch {
+            return href;
+        }
+    }
+    /**
+     * Check if we should finish crawling
+     */
+    shouldFinish(observation) {
+        if (this.pagesVisited >= this.config.maxPages)
+            return true;
+        if (this.visitQueue.length === 0 && observation.clickableElements.length === 0)
+            return true;
+        return false;
+    }
+    /**
+     * Get reason for finishing
+     */
+    getFinishReason() {
+        if (this.pagesVisited >= this.config.maxPages)
+            return `Reached maximum page limit (${this.config.maxPages})`;
+        if (this.visitQueue.length === 0)
+            return 'No more URLs to visit in the queue';
+        return 'Crawling complete';
+    }
+    /**
+     * Get crawling statistics
+     */
+    getStats() {
+        return {
+            pagesVisited: this.pagesVisited,
+            queueSize: this.visitQueue.length,
+            visitedUrls: Array.from(this.visited),
+            currentDepth: this.currentDepth,
+            maxDepth: this.config.maxDepth,
+            maxPages: this.config.maxPages
+        };
+    }
+}

package/lib/program.js

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) Microsoft Corporation.
+ * Copyright (c) DarbotLabs.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,10 +50,19 @@ program
     .option('--user-data-dir <path>', 'path to the user data directory. If not specified, a temporary directory will be created.')
     .option('--viewport-size <size>', 'specify browser viewport size in pixels, for example "1280, 720"')
     .option('--vision', 'Run server that uses screenshots (Aria snapshots are used by default)')
+    .option('--edge-profile <name>', 'Edge profile name to use (e.g., "Profile 1", "Default"). This is recorded in saved session states.')
+    .option('--edge-profile-email <email>', 'Email associated with the Edge profile. This is recorded in saved session states for context.')
+    .option('--auto-sign-in', 'Auto sign in with work/school account (Edge profile preference)')
+    .option('--profile-switching', 'Enable automatic profile switching based on site (Edge profile preference)')
+    .option('--intranet-switch', 'Automatically switch to work profile for intranet sites (Edge profile preference)')
+    .option('--ie-mode-switch', 'Automatically switch profile for IE mode sites (Edge profile preference)')
+    .option('--default-profile <name>', 'Default Edge profile for external links (Edge profile preference)')
     .addOption(new Option('--extension', 'Allow connecting to a running browser instance (Edge/Chrome only). Requires the \'Darbot Browser MCP\' browser extension to be installed.').hideHelp())
     .action(async (options) => {
     const config = await resolveCLIConfig(options);
-    const httpServer = config.server.port !== undefined ? await startHttpServer(config.server) : undefined;
+    const httpResult = config.server.port !== undefined ? await startHttpServer(config.server) : undefined;
+    const httpServer = httpResult?.httpServer;
+    const expressApp = httpResult?.app;
     if (config.extension) {
         if (!httpServer)
             throw new Error('--port parameter is required for extension mode');
@@ -61,9 +70,12 @@ program
         config.browser.cdpEndpoint = await startCDPRelayServer(httpServer);
     }
     const server = new Server(config);
-    server.setupExitWatchdog();
-    if (httpServer)
-        await startHttpTransport(httpServer, server);
+    // Use http mode when port is specified to prevent exit on stdin close
+    server.setupExitWatchdog(httpServer ? 'http' : 'stdio');
+    if (httpServer && expressApp)
+        await startHttpTransport(httpServer, server, expressApp);
+    else if (httpServer)
+        throw new Error('Express app not initialized');
     else
         await startStdioTransport(server);
     if (config.saveTrace) {