@darbotlabs/darbot-browser-mcp 0.1.1 → 1.3.0

package/lib/ai/manualPromise.js

@@ -0,0 +1,111 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class ManualPromise extends Promise {
+    _resolve;
+    _reject;
+    _isDone;
+    constructor() {
+        let resolve;
+        let reject;
+        super((f, r) => {
+            resolve = f;
+            reject = r;
+        });
+        this._isDone = false;
+        this._resolve = resolve;
+        this._reject = reject;
+    }
+    isDone() {
+        return this._isDone;
+    }
+    resolve(t) {
+        this._isDone = true;
+        this._resolve(t);
+    }
+    reject(e) {
+        this._isDone = true;
+        this._reject(e);
+    }
+    static get [Symbol.species]() {
+        return Promise;
+    }
+    get [Symbol.toStringTag]() {
+        return 'ManualPromise';
+    }
+}
+export class LongStandingScope {
+    _terminateError;
+    _closeError;
+    _terminatePromises = new Map();
+    _isClosed = false;
+    reject(error) {
+        this._isClosed = true;
+        this._terminateError = error;
+        for (const p of this._terminatePromises.keys())
+            p.resolve(error);
+    }
+    close(error) {
+        this._isClosed = true;
+        this._closeError = error;
+        for (const [p, frames] of this._terminatePromises)
+            p.resolve(cloneError(error, frames));
+    }
+    isClosed() {
+        return this._isClosed;
+    }
+    static async raceMultiple(scopes, promise) {
+        return Promise.race(scopes.map(s => s.race(promise)));
+    }
+    async race(promise) {
+        return this._race(Array.isArray(promise) ? promise : [promise], false);
+    }
+    async safeRace(promise, defaultValue) {
+        return this._race([promise], true, defaultValue);
+    }
+    async _race(promises, safe, defaultValue) {
+        const terminatePromise = new ManualPromise();
+        const frames = captureRawStack();
+        if (this._terminateError)
+            terminatePromise.resolve(this._terminateError);
+        if (this._closeError)
+            terminatePromise.resolve(cloneError(this._closeError, frames));
+        this._terminatePromises.set(terminatePromise, frames);
+        try {
+            return await Promise.race([
+                terminatePromise.then(e => safe ? defaultValue : Promise.reject(e)),
+                ...promises
+            ]);
+        }
+        finally {
+            this._terminatePromises.delete(terminatePromise);
+        }
+    }
+}
+function cloneError(error, frames) {
+    const clone = new Error();
+    clone.name = error.name;
+    clone.message = error.message;
+    clone.stack = [error.name + ':' + error.message, ...frames].join('\n');
+    return clone;
+}
+function captureRawStack() {
+    const stackTraceLimit = Error.stackTraceLimit;
+    Error.stackTraceLimit = 50;
+    const error = new Error();
+    const stack = error.stack || '';
+    Error.stackTraceLimit = stackTraceLimit;
+    return stack.split('\n');
+}

package/lib/ai/memory.js

@@ -0,0 +1,273 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import debug from 'debug';
+const log = debug('darbot:memory');
+/**
+ * Local file-based memory storage implementation
+ */
+export class LocalMemoryStorage {
+    storagePath;
+    maxStates;
+    constructor(config = {}) {
+        this.storagePath = config.storagePath || path.join(process.cwd(), '.darbot', 'memory');
+        this.maxStates = config.maxStates || 1000;
+        this.ensureStorageDirectory();
+    }
+    ensureStorageDirectory() {
+        if (!fs.existsSync(this.storagePath))
+            fs.mkdirSync(this.storagePath, { recursive: true });
+    }
+    getStatePath(stateHash) {
+        return path.join(this.storagePath, `${stateHash}.json`);
+    }
+    async storeState(state) {
+        try {
+            const statePath = this.getStatePath(state.stateHash);
+            await fs.promises.writeFile(statePath, JSON.stringify(state, null, 2));
+            log('Stored state:', state.stateHash, state.url);
+            // Clean up old states if we exceed the limit
+            await this.cleanupOldStates();
+        }
+        catch (error) {
+            log('Error storing state:', error);
+            throw error;
+        }
+    }
+    async getState(stateHash) {
+        try {
+            const statePath = this.getStatePath(stateHash);
+            if (!fs.existsSync(statePath))
+                return null;
+            const data = await fs.promises.readFile(statePath, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch (error) {
+            log('Error reading state:', error);
+            return null;
+        }
+    }
+    async hasState(stateHash) {
+        const statePath = this.getStatePath(stateHash);
+        return fs.existsSync(statePath);
+    }
+    async getAllStates() {
+        try {
+            const files = await fs.promises.readdir(this.storagePath);
+            const states = [];
+            for (const file of files) {
+                if (file.endsWith('.json')) {
+                    const filePath = path.join(this.storagePath, file);
+                    try {
+                        const data = await fs.promises.readFile(filePath, 'utf-8');
+                        states.push(JSON.parse(data));
+                    }
+                    catch (error) {
+                        log('Error reading state file:', file, error);
+                    }
+                }
+            }
+            return states.sort((a, b) => a.timestamp - b.timestamp);
+        }
+        catch (error) {
+            log('Error reading states:', error);
+            return [];
+        }
+    }
+    async getUnvisitedLinks() {
+        const states = await this.getAllStates();
+        const visited = new Set(states.filter(s => s.visited).map(s => s.url));
+        const allLinks = new Set();
+        states.forEach(state => {
+            state.links.forEach(link => {
+                if (!visited.has(link))
+                    allLinks.add(link);
+            });
+        });
+        return Array.from(allLinks);
+    }
+    async clear() {
+        try {
+            const files = await fs.promises.readdir(this.storagePath);
+            await Promise.all(files.map(file => fs.promises.unlink(path.join(this.storagePath, file))));
+            log('Cleared memory storage');
+        }
+        catch (error) {
+            log('Error clearing storage:', error);
+            throw error;
+        }
+    }
+    async cleanupOldStates() {
+        const states = await this.getAllStates();
+        if (states.length <= this.maxStates)
+            return;
+        // Remove oldest states
+        const toRemove = states.slice(0, states.length - this.maxStates);
+        await Promise.all(toRemove.map(state => fs.promises.unlink(this.getStatePath(state.stateHash)).catch(() => { })));
+        log(`Cleaned up ${toRemove.length} old states`);
+    }
+}
+/**
+ * Darbot Memory MCP connector (placeholder for future implementation)
+ */
+export class DarbotMemoryStorage {
+    constructor(config = {}) {
+        // TODO: Implement darbot-memory-mcp integration
+        log('Darbot Memory MCP connector not yet implemented, falling back to local storage');
+    }
+    async storeState(state) {
+        // TODO: Send to darbot-memory-mcp server
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+    async getState(stateHash) {
+        // TODO: Query darbot-memory-mcp server
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+    async hasState(stateHash) {
+        // TODO: Check darbot-memory-mcp server
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+    async getAllStates() {
+        // TODO: Fetch from darbot-memory-mcp server
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+    async getUnvisitedLinks() {
+        // TODO: Query darbot-memory-mcp server
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+    async clear() {
+        // TODO: Clear darbot-memory-mcp storage
+        throw new Error('Darbot Memory MCP connector not yet implemented');
+    }
+}
+/**
+ * Memory manager with optional darbot-memory-mcp integration
+ */
+export class MemoryManager {
+    storage;
+    config;
+    constructor(config = { enabled: true }) {
+        this.config = config;
+        if (!config.enabled) {
+            this.storage = new LocalMemoryStorage(); // Dummy storage that won't be used
+            return;
+        }
+        switch (config.connector) {
+            case 'darbot-memory-mcp':
+                try {
+                    this.storage = new DarbotMemoryStorage();
+                }
+                catch (error) {
+                    log('Failed to initialize darbot-memory-mcp connector, falling back to local storage:', error);
+                    this.storage = new LocalMemoryStorage({
+                        storagePath: config.storagePath,
+                        maxStates: config.maxStates
+                    });
+                }
+                break;
+            case 'local':
+            default:
+                this.storage = new LocalMemoryStorage({
+                    storagePath: config.storagePath,
+                    maxStates: config.maxStates
+                });
+                break;
+        }
+    }
+    /**
+     * Generate a hash for the current page state
+     */
+    static stateHash(domSnapshot) {
+        return crypto.createHash('sha256').update(domSnapshot).digest('hex').substring(0, 16);
+    }
+    /**
+     * Store a crawl state with screenshot
+     */
+    async storeState(url, title, domSnapshot, screenshot, links = []) {
+        if (!this.config.enabled)
+            return '';
+        const stateHash = MemoryManager.stateHash(domSnapshot);
+        let screenshotPath;
+        // Save screenshot if provided
+        if (screenshot) {
+            const screenshotDir = path.join(process.cwd(), '.darbot', 'screenshots');
+            if (!fs.existsSync(screenshotDir))
+                fs.mkdirSync(screenshotDir, { recursive: true });
+            screenshotPath = path.join(screenshotDir, `${stateHash}.png`);
+            await fs.promises.writeFile(screenshotPath, screenshot);
+        }
+        const state = {
+            url,
+            title,
+            stateHash,
+            timestamp: Date.now(),
+            screenshot: screenshotPath,
+            links,
+            visited: true
+        };
+        await this.storage.storeState(state);
+        return stateHash;
+    }
+    /**
+     * Check if we've seen this state before
+     */
+    async hasState(domSnapshot) {
+        if (!this.config.enabled)
+            return false;
+        const stateHash = MemoryManager.stateHash(domSnapshot);
+        return await this.storage.hasState(stateHash);
+    }
+    /**
+     * Get a stored state by hash
+     */
+    async getState(stateHash) {
+        if (!this.config.enabled)
+            return null;
+        return await this.storage.getState(stateHash);
+    }
+    /**
+     * Get all stored states
+     */
+    async getAllStates() {
+        if (!this.config.enabled)
+            return [];
+        return await this.storage.getAllStates();
+    }
+    /**
+     * Get unvisited links for BFS crawling
+     */
+    async getUnvisitedLinks() {
+        if (!this.config.enabled)
+            return [];
+        return await this.storage.getUnvisitedLinks();
+    }
+    /**
+     * Clear all stored states
+     */
+    async clear() {
+        if (!this.config.enabled)
+            return;
+        await this.storage.clear();
+    }
+    /**
+     * Check if memory is enabled
+     */
+    get enabled() {
+        return this.config.enabled;
+    }
+}

package/lib/ai/ml-scorer.js

@@ -0,0 +1,265 @@
+/**
+ * Copyright (c) DarbotLabs.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * ML-based scoring system for intelligent URL and element prioritization
+ *
+ * This replaces basic heuristics with feature-based scoring that learns from
+ * crawl patterns and goal descriptions to make better decisions.
+ */
+import debug from 'debug';
+const log = debug('darbot:ml-scorer');
+/**
+ * ML-inspired scoring system using feature engineering and weighted scoring
+ */
+export class MLBasedScorer {
+    weights;
+    goalKeywords;
+    learnedPatterns = new Map();
+    constructor(goal) {
+        // Initialize feature weights (these could be learned from data)
+        this.weights = new Map([
+            // URL structure weights
+            ['urlDepth', -0.3], // Prefer shallower URLs
+            ['urlLength', -0.1], // Prefer shorter URLs
+            ['pathSegments', 0.2], // More segments = more specific
+            ['queryParams', -0.2], // Too many params = dynamic/session pages
+            // Content weights
+            ['textLength', 0.3], // More text = more content
+            ['hasKeywords', 2.0], // Strong signal for goal relevance
+            ['semanticRelevance', 1.5], // Goal-based relevance
+            // Context weights
+            ['parentScore', 0.4], // Good parents suggest good children
+            ['visitedSiblings', -0.2], // Avoid repetitive sibling pages
+            ['domDepth', -0.1], // Prefer more accessible elements
+            // Pattern weights
+            ['contentPattern', 1.0], // Content pages
+            ['navPattern', 0.5], // Navigation pages
+            ['utilityPattern', -0.5], // Utility pages (login, etc.)
+        ]);
+        // Extract keywords from goal
+        this.goalKeywords = new Set();
+        if (goal) {
+            const words = goal.toLowerCase()
+                .split(/\s+/)
+                .filter(w => w.length > 3)
+                .filter(w => !this.isStopWord(w));
+            words.forEach(w => this.goalKeywords.add(w));
+        }
+        log('Initialized ML scorer with goal keywords:', Array.from(this.goalKeywords));
+    }
+    /**
+     * Score a URL for crawling priority
+     */
+    scoreUrl(url, context) {
+        const features = this.extractUrlFeatures(url, context);
+        return this.calculateScore(features);
+    }
+    /**
+     * Score an element for interaction priority
+     */
+    scoreElement(element, context) {
+        const features = this.extractElementFeatures(element, context);
+        return this.calculateScore(features);
+    }
+    /**
+     * Update learned patterns based on successful crawl
+     */
+    learn(url, success, features) {
+        // Extract pattern from URL
+        const pattern = this.extractPattern(url);
+        // Update pattern score based on success
+        const currentScore = this.learnedPatterns.get(pattern) || 0;
+        const delta = success ? 0.1 : -0.05;
+        this.learnedPatterns.set(pattern, currentScore + delta);
+        log('Updated pattern score:', pattern, '->', this.learnedPatterns.get(pattern));
+    }
+    /**
+     * Extract features from URL
+     */
+    extractUrlFeatures(url, context) {
+        const features = new Map();
+        try {
+            const urlObj = new URL(url);
+            const path = urlObj.pathname;
+            // URL structure features
+            features.set('urlDepth', context.currentDepth);
+            features.set('urlLength', url.length / 100); // Normalize
+            features.set('hasNumbers', /\d/.test(path) ? 1 : 0);
+            features.set('hasHyphens', /-/.test(path) ? 1 : 0);
+            features.set('pathSegments', path.split('/').filter(Boolean).length);
+            features.set('queryParams', urlObj.searchParams.size);
+            // Pattern matching
+            features.set('contentPattern', this.matchesContentPattern(url) ? 1 : 0);
+            features.set('navPattern', this.matchesNavPattern(url) ? 1 : 0);
+            features.set('utilityPattern', this.matchesUtilityPattern(url) ? 1 : 0);
+            // Goal relevance
+            const relevance = this.calculateSemanticRelevance(url);
+            features.set('semanticRelevance', relevance);
+            features.set('hasKeywords', relevance > 0.5 ? 1 : 0);
+            // Context features
+            const parentPattern = context.parentUrl ? this.extractPattern(context.parentUrl) : '';
+            features.set('parentScore', this.learnedPatterns.get(parentPattern) || 0);
+            // Visited sibling counting
+            const urlPattern = this.extractPattern(url);
+            let visitedSiblings = 0;
+            for (const visitedUrl of context.visitedUrls) {
+                if (this.extractPattern(visitedUrl) === urlPattern)
+                    visitedSiblings++;
+            }
+            features.set('visitedSiblings', visitedSiblings);
+        }
+        catch (error) {
+            log('Error extracting URL features:', error);
+        }
+        return features;
+    }
+    /**
+     * Extract features from element
+     */
+    extractElementFeatures(element, context) {
+        const features = new Map();
+        const text = element.text.toLowerCase();
+        // Text features
+        features.set('textLength', Math.min(element.text.length / 50, 1));
+        features.set('hasKeywords', this.containsKeywords(text) ? 1 : 0);
+        // Semantic relevance
+        features.set('semanticRelevance', this.calculateSemanticRelevance(text));
+        // DOM features
+        const selectorDepth = element.selector.split('>').length;
+        features.set('domDepth', selectorDepth);
+        // Element type features
+        const isActionButton = ['button', 'submit'].includes(element.tag);
+        features.set('isActionButton', isActionButton ? 1 : 0);
+        // Pattern matching
+        features.set('contentPattern', this.matchesContentPattern(text) ? 1 : 0);
+        features.set('navPattern', this.matchesNavPattern(text) ? 1 : 0);
+        features.set('utilityPattern', this.matchesUtilityPattern(text) ? 1 : 0);
+        return features;
+    }
+    /**
+     * Calculate final score from features
+     */
+    calculateScore(features) {
+        let score = 0;
+        for (const [feature, value] of features.entries()) {
+            const weight = this.weights.get(feature) || 0;
+            score += weight * value;
+        }
+        // Apply sigmoid to bound score between 0 and 1
+        return 1 / (1 + Math.exp(-score));
+    }
+    /**
+     * Calculate semantic relevance to goal
+     */
+    calculateSemanticRelevance(text) {
+        if (this.goalKeywords.size === 0)
+            return 0.5; // Neutral if no goal specified
+        const words = text.toLowerCase().split(/\W+/);
+        let matches = 0;
+        for (const word of words) {
+            if (this.goalKeywords.has(word))
+                matches++;
+        }
+        // Normalize by goal keyword count
+        return Math.min(matches / this.goalKeywords.size, 1.0);
+    }
+    /**
+     * Check if text contains goal keywords
+     */
+    containsKeywords(text) {
+        if (this.goalKeywords.size === 0)
+            return false;
+        const words = text.toLowerCase().split(/\W+/);
+        return words.some(word => this.goalKeywords.has(word));
+    }
+    /**
+     * Extract pattern from URL
+     */
+    extractPattern(url) {
+        try {
+            const urlObj = new URL(url);
+            // Pattern = domain + path structure (without specific IDs/numbers)
+            const pathPattern = urlObj.pathname.replace(/\d+/g, '*').replace(/\/[a-f0-9-]{36}/gi, '/*');
+            return `${urlObj.hostname}${pathPattern}`;
+        }
+        catch {
+            return url;
+        }
+    }
+    /**
+     * Check if URL/text matches content patterns
+     */
+    matchesContentPattern(text) {
+        const patterns = [
+            /article/i, /post/i, /blog/i, /news/i, /story/i,
+            /product/i, /item/i, /detail/i, /content/i, /page/i,
+            /documentation/i, /docs/i, /guide/i, /tutorial/i
+        ];
+        return patterns.some(p => p.test(text));
+    }
+    /**
+     * Check if URL/text matches navigation patterns
+     */
+    matchesNavPattern(text) {
+        const patterns = [
+            /category/i, /section/i, /menu/i, /nav/i,
+            /index/i, /list/i, /archive/i, /browse/i
+        ];
+        return patterns.some(p => p.test(text));
+    }
+    /**
+     * Check if URL/text matches utility patterns (usually low priority)
+     */
+    matchesUtilityPattern(text) {
+        const patterns = [
+            /login/i, /signin/i, /register/i, /signup/i,
+            /logout/i, /signout/i, /profile/i, /account/i,
+            /terms/i, /privacy/i, /legal/i, /cookie/i,
+            /contact/i, /about/i, /help/i, /faq/i
+        ];
+        return patterns.some(p => p.test(text));
+    }
+    /**
+     * Check if word is a stop word
+     */
+    isStopWord(word) {
+        const stopWords = new Set([
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'up', 'about', 'into', 'through', 'during'
+        ]);
+        return stopWords.has(word.toLowerCase());
+    }
+    /**
+     * Get learned patterns for debugging/export
+     */
+    getLearnedPatterns() {
+        return new Map(this.learnedPatterns);
+    }
+    /**
+     * Export scoring statistics
+     */
+    getStatistics() {
+        const scores = Array.from(this.learnedPatterns.values());
+        const avgScore = scores.length > 0
+            ? scores.reduce((a, b) => a + b, 0) / scores.length
+            : 0;
+        return {
+            goalKeywords: Array.from(this.goalKeywords),
+            learnedPatterns: this.learnedPatterns.size,
+            averagePatternScore: avgScore
+        };
+    }
+}