brave-real-browser-mcp-server 2.7.6 → 2.8.0

package/dist/extractors/content-type-extractors.js

@@ -0,0 +1,314 @@
+/**
+ * Image Scraper - Extract all images with metadata
+ */
+export async function scrapeImages(page, selector) {
+    return await page.evaluate((sel) => {
+        const images = sel
+            ? document.querySelectorAll(sel)
+            : document.querySelectorAll('img');
+        const results = [];
+        images.forEach((img) => {
+            const imgEl = img;
+            const rect = imgEl.getBoundingClientRect();
+            const styles = window.getComputedStyle(imgEl);
+            results.push({
+                src: imgEl.src || imgEl.getAttribute('src') || '',
+                alt: imgEl.alt || '',
+                title: imgEl.title || '',
+                width: imgEl.width || rect.width,
+                height: imgEl.height || rect.height,
+                naturalWidth: imgEl.naturalWidth,
+                naturalHeight: imgEl.naturalHeight,
+                loading: imgEl.loading || 'auto',
+                srcset: imgEl.srcset || '',
+                sizes: imgEl.sizes || '',
+                isVisible: styles.display !== 'none' && styles.visibility !== 'hidden' && rect.width > 0 && rect.height > 0
+            });
+        });
+        return results;
+    }, selector || null);
+}
+/**
+ * Link Harvester - Extract all links with classification
+ */
+export async function harvestLinks(page, options) {
+    const opts = {
+        includeInternal: true,
+        includeExternal: true,
+        includeAnchors: true,
+        ...options
+    };
+    return await page.evaluate((config) => {
+        const currentDomain = window.location.hostname;
+        const internal = [];
+        const external = [];
+        const anchors = [];
+        const all = [];
+        const links = document.querySelectorAll('a[href]');
+        links.forEach((link) => {
+            const href = link.href;
+            const text = link.innerText.trim();
+            const title = link.getAttribute('title') || '';
+            const target = link.getAttribute('target') || '';
+            if (!href)
+                return;
+            // Anchor links
+            if (href.startsWith('#')) {
+                if (config.includeAnchors) {
+                    anchors.push({ href, text, target });
+                    all.push({ href, text, type: 'anchor' });
+                }
+                return;
+            }
+            try {
+                const url = new URL(href);
+                // Internal vs External
+                if (url.hostname === currentDomain || url.hostname === '') {
+                    if (config.includeInternal) {
+                        internal.push({ href, text, title });
+                        all.push({ href, text, type: 'internal' });
+                    }
+                }
+                else {
+                    if (config.includeExternal) {
+                        external.push({ href, text, title });
+                        all.push({ href, text, type: 'external' });
+                    }
+                }
+            }
+            catch (e) {
+                // Invalid URL, treat as internal
+                if (config.includeInternal) {
+                    internal.push({ href, text, title });
+                    all.push({ href, text, type: 'internal' });
+                }
+            }
+        });
+        return { internal, external, anchors, all };
+    }, opts);
+}
+/**
+ * Media Extractor - Extract videos, audio files, and embedded media
+ */
+export async function extractMedia(page) {
+    return await page.evaluate(() => {
+        const videos = [];
+        const audio = [];
+        const iframes = [];
+        const embeds = [];
+        // Extract video elements
+        document.querySelectorAll('video').forEach((video) => {
+            const sources = [];
+            video.querySelectorAll('source').forEach((source) => {
+                sources.push(source.src);
+            });
+            videos.push({
+                src: video.src || sources[0] || '',
+                sources: sources,
+                poster: video.poster || '',
+                width: video.width,
+                height: video.height,
+                controls: video.controls,
+                autoplay: video.autoplay,
+                loop: video.loop,
+                muted: video.muted,
+                duration: video.duration,
+                currentTime: video.currentTime
+            });
+        });
+        // Extract audio elements
+        document.querySelectorAll('audio').forEach((audioEl) => {
+            const sources = [];
+            audioEl.querySelectorAll('source').forEach((source) => {
+                sources.push(source.src);
+            });
+            audio.push({
+                src: audioEl.src || sources[0] || '',
+                sources: sources,
+                controls: audioEl.controls,
+                autoplay: audioEl.autoplay,
+                loop: audioEl.loop,
+                muted: audioEl.muted,
+                duration: audioEl.duration
+            });
+        });
+        // Extract iframes
+        document.querySelectorAll('iframe').forEach((iframe) => {
+            const src = iframe.src;
+            let platform = 'unknown';
+            // Detect common video platforms
+            if (src.includes('youtube.com') || src.includes('youtu.be')) {
+                platform = 'youtube';
+            }
+            else if (src.includes('vimeo.com')) {
+                platform = 'vimeo';
+            }
+            else if (src.includes('dailymotion.com')) {
+                platform = 'dailymotion';
+            }
+            else if (src.includes('facebook.com')) {
+                platform = 'facebook';
+            }
+            else if (src.includes('twitter.com') || src.includes('x.com')) {
+                platform = 'twitter';
+            }
+            iframes.push({
+                src: src,
+                title: iframe.title || '',
+                width: iframe.width,
+                height: iframe.height,
+                platform: platform,
+                allowFullscreen: iframe.allowFullscreen
+            });
+        });
+        // Extract embed elements
+        document.querySelectorAll('embed, object').forEach((embed) => {
+            embeds.push({
+                src: embed.getAttribute('src') || embed.getAttribute('data') || '',
+                type: embed.getAttribute('type') || '',
+                width: embed.getAttribute('width') || '',
+                height: embed.getAttribute('height') || ''
+            });
+        });
+        return { videos, audio, iframes, embeds };
+    });
+}
+/**
+ * PDF Link Finder - Find all downloadable file links
+ */
+export async function findDownloadableFiles(page) {
+    return await page.evaluate(() => {
+        const pdfs = [];
+        const documents = [];
+        const archives = [];
+        const images = [];
+        const other = [];
+        const links = document.querySelectorAll('a[href]');
+        links.forEach((link) => {
+            const href = link.href;
+            const text = link.innerText.trim();
+            const download = link.getAttribute('download');
+            if (!href)
+                return;
+            const url = href.toLowerCase();
+            const fileInfo = { href, text, size: link.getAttribute('data-size') || undefined };
+            // PDF files
+            if (url.endsWith('.pdf') || url.includes('.pdf?') || download?.endsWith('.pdf')) {
+                pdfs.push(fileInfo);
+            }
+            // Document files
+            else if (url.match(/\.(doc|docx|xls|xlsx|ppt|pptx|odt|ods|odp)($|\?)/)) {
+                const match = url.match(/\.(doc|docx|xls|xlsx|ppt|pptx|odt|ods|odp)($|\?)/);
+                documents.push({ ...fileInfo, type: match ? match[1] : 'unknown' });
+            }
+            // Archive files
+            else if (url.match(/\.(zip|rar|7z|tar|gz|bz2)($|\?)/)) {
+                const match = url.match(/\.(zip|rar|7z|tar|gz|bz2)($|\?)/);
+                archives.push({ ...fileInfo, type: match ? match[1] : 'unknown' });
+            }
+            // Image files (downloadable)
+            else if (url.match(/\.(jpg|jpeg|png|gif|bmp|svg|webp|ico)($|\?)/) && download) {
+                const match = url.match(/\.(jpg|jpeg|png|gif|bmp|svg|webp|ico)($|\?)/);
+                images.push({ ...fileInfo, type: match ? match[1] : 'unknown' });
+            }
+            // Other downloadable files
+            else if (download || url.match(/\.(exe|dmg|apk|deb|rpm|msi|iso)($|\?)/)) {
+                const match = url.match(/\.([a-z0-9]+)($|\?)/);
+                other.push({ ...fileInfo, type: match ? match[1] : 'unknown' });
+            }
+        });
+        return { pdfs, documents, archives, images, other };
+    });
+}
+/**
+ * Social Media Links Extractor - Extract social media profile links
+ */
+export async function extractSocialLinks(page) {
+    return await page.evaluate(() => {
+        const socialLinks = {
+            facebook: [],
+            twitter: [],
+            instagram: [],
+            linkedin: [],
+            youtube: [],
+            github: [],
+            pinterest: [],
+            tiktok: [],
+            other: []
+        };
+        const links = document.querySelectorAll('a[href]');
+        links.forEach((link) => {
+            const href = link.href.toLowerCase();
+            if (href.includes('facebook.com')) {
+                socialLinks.facebook.push(link.href);
+            }
+            else if (href.includes('twitter.com') || href.includes('x.com')) {
+                socialLinks.twitter.push(link.href);
+            }
+            else if (href.includes('instagram.com')) {
+                socialLinks.instagram.push(link.href);
+            }
+            else if (href.includes('linkedin.com')) {
+                socialLinks.linkedin.push(link.href);
+            }
+            else if (href.includes('youtube.com') || href.includes('youtu.be')) {
+                socialLinks.youtube.push(link.href);
+            }
+            else if (href.includes('github.com')) {
+                socialLinks.github.push(link.href);
+            }
+            else if (href.includes('pinterest.com')) {
+                socialLinks.pinterest.push(link.href);
+            }
+            else if (href.includes('tiktok.com')) {
+                socialLinks.tiktok.push(link.href);
+            }
+        });
+        // Remove duplicates
+        Object.keys(socialLinks).forEach((key) => {
+            socialLinks[key] = Array.from(new Set(socialLinks[key]));
+        });
+        return socialLinks;
+    });
+}
+/**
+ * Email and Phone Extractor - Extract contact information from page
+ */
+export async function extractContactInfo(page) {
+    return await page.evaluate(() => {
+        const text = document.body.innerText;
+        const emails = [];
+        const phones = [];
+        const addresses = [];
+        // Extract emails
+        const emailRegex = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
+        const emailMatches = text.match(emailRegex);
+        if (emailMatches) {
+            emails.push(...emailMatches);
+        }
+        // Also check mailto links
+        document.querySelectorAll('a[href^="mailto:"]').forEach((link) => {
+            const email = link.href.replace('mailto:', '').split('?')[0];
+            if (email)
+                emails.push(email);
+        });
+        // Extract phone numbers (various formats)
+        const phoneRegex = /(\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/g;
+        const phoneMatches = text.match(phoneRegex);
+        if (phoneMatches) {
+            phones.push(...phoneMatches);
+        }
+        // Also check tel links
+        document.querySelectorAll('a[href^="tel:"]').forEach((link) => {
+            const phone = link.href.replace('tel:', '');
+            if (phone)
+                phones.push(phone);
+        });
+        // Remove duplicates
+        return {
+            emails: Array.from(new Set(emails)),
+            phones: Array.from(new Set(phones)),
+            addresses: addresses
+        };
+    });
+}

package/dist/extractors/extractors.test.js

@@ -0,0 +1,17 @@
+// Basic tests for extractor modules
+import { describe, it, expect } from 'vitest';
+describe('Smart Data Extractors', () => {
+    it('should exist', () => {
+        expect(true).toBe(true);
+    });
+});
+describe('Multi-Element Extractors', () => {
+    it('should exist', () => {
+        expect(true).toBe(true);
+    });
+});
+describe('Content Type Extractors', () => {
+    it('should exist', () => {
+        expect(true).toBe(true);
+    });
+});

package/dist/extractors/multi-element-extractors.js

@@ -0,0 +1,325 @@
+/**
+ * Batch Element Scraper - Extract multiple similar elements (products, articles, etc.)
+ */
+export async function batchScrapeElements(page, containerSelector, fields) {
+    return await page.evaluate((container, fieldMap) => {
+        const containers = document.querySelectorAll(container);
+        const results = [];
+        containers.forEach((containerEl) => {
+            const item = {};
+            Object.entries(fieldMap).forEach(([fieldName, selector]) => {
+                const element = containerEl.querySelector(selector);
+                if (element) {
+                    // Extract different types of data based on element type
+                    if (element.tagName === 'IMG') {
+                        item[fieldName] = {
+                            src: element.src,
+                            alt: element.alt,
+                            title: element.getAttribute('title') || ''
+                        };
+                    }
+                    else if (element.tagName === 'A') {
+                        item[fieldName] = {
+                            text: element.innerText.trim(),
+                            href: element.href
+                        };
+                    }
+                    else if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
+                        item[fieldName] = element.value;
+                    }
+                    else {
+                        item[fieldName] = element.innerText.trim();
+                    }
+                }
+                else {
+                    item[fieldName] = null;
+                }
+            });
+            // Only add if at least some fields were found
+            const hasData = Object.values(item).some(v => v !== null);
+            if (hasData) {
+                results.push(item);
+            }
+        });
+        return results;
+    }, containerSelector, fields);
+}
+/**
+ * Nested Data Extraction - Extract hierarchical data maintaining parent-child relationships
+ */
+export async function extractNestedData(page, parentSelector, childSelector, options) {
+    const opts = {
+        parentFields: {},
+        childFields: {},
+        maxDepth: 3,
+        ...options
+    };
+    return await page.evaluate((parentSel, childSel, config) => {
+        const results = [];
+        const parents = document.querySelectorAll(parentSel);
+        parents.forEach((parentEl) => {
+            const parentData = {};
+            // Extract parent fields
+            if (Object.keys(config.parentFields).length > 0) {
+                Object.entries(config.parentFields).forEach(([fieldName, selector]) => {
+                    const element = parentEl.querySelector(selector);
+                    if (element) {
+                        parentData[fieldName] = element.innerText.trim();
+                    }
+                });
+            }
+            else {
+                // Default: extract text content
+                parentData.content = parentEl.innerText.trim();
+            }
+            // Extract children
+            const children = [];
+            const childElements = parentEl.querySelectorAll(childSel);
+            childElements.forEach((childEl) => {
+                const childData = {};
+                if (Object.keys(config.childFields).length > 0) {
+                    Object.entries(config.childFields).forEach(([fieldName, selector]) => {
+                        const element = childEl.querySelector(selector);
+                        if (element) {
+                            childData[fieldName] = element.innerText.trim();
+                        }
+                    });
+                }
+                else {
+                    // Default: extract text content
+                    childData.content = childEl.innerText.trim();
+                }
+                if (Object.keys(childData).length > 0) {
+                    children.push(childData);
+                }
+            });
+            results.push({
+                parent: parentData,
+                children: children
+            });
+        });
+        return results;
+    }, parentSelector, childSelector, opts);
+}
+/**
+ * Attribute Harvester - Collect all attributes from elements
+ */
+export async function harvestAttributes(page, selector, attributes) {
+    return await page.evaluate((sel, attrs) => {
+        const elements = document.querySelectorAll(sel);
+        const results = [];
+        elements.forEach((element) => {
+            const attrData = {};
+            if (attrs && attrs.length > 0) {
+                // Collect specific attributes
+                attrs.forEach((attrName) => {
+                    const value = element.getAttribute(attrName);
+                    if (value !== null) {
+                        attrData[attrName] = value;
+                    }
+                });
+            }
+            else {
+                // Collect all attributes
+                Array.from(element.attributes).forEach((attr) => {
+                    attrData[attr.name] = attr.value;
+                });
+            }
+            // Add element tag name and text content
+            attrData._tagName = element.tagName.toLowerCase();
+            attrData._textContent = element.innerText?.trim() || '';
+            results.push(attrData);
+        });
+        return results;
+    }, selector, attributes || null);
+}
+/**
+ * Deep Element Scraper - Extract elements with all their properties and computed styles
+ */
+export async function deepScrapeElements(page, selector, options) {
+    const opts = {
+        includeStyles: false,
+        includePosition: true,
+        includeVisibility: true,
+        ...options
+    };
+    return await page.evaluate((sel, config) => {
+        const elements = document.querySelectorAll(sel);
+        const results = [];
+        elements.forEach((element) => {
+            const data = {
+                tagName: element.tagName.toLowerCase(),
+                id: element.id || null,
+                classes: Array.from(element.classList),
+                textContent: element.innerText?.trim() || '',
+                attributes: {}
+            };
+            // Collect all attributes
+            Array.from(element.attributes).forEach((attr) => {
+                data.attributes[attr.name] = attr.value;
+            });
+            // Get position and dimensions
+            if (config.includePosition) {
+                const rect = element.getBoundingClientRect();
+                data.position = {
+                    top: rect.top,
+                    left: rect.left,
+                    width: rect.width,
+                    height: rect.height,
+                    right: rect.right,
+                    bottom: rect.bottom
+                };
+            }
+            // Get visibility information
+            if (config.includeVisibility) {
+                const styles = window.getComputedStyle(element);
+                data.visibility = {
+                    display: styles.display,
+                    visibility: styles.visibility,
+                    opacity: styles.opacity,
+                    isVisible: styles.display !== 'none' &&
+                        styles.visibility !== 'hidden' &&
+                        parseFloat(styles.opacity) > 0
+                };
+            }
+            // Get computed styles
+            if (config.includeStyles) {
+                const styles = window.getComputedStyle(element);
+                data.styles = {
+                    color: styles.color,
+                    backgroundColor: styles.backgroundColor,
+                    fontSize: styles.fontSize,
+                    fontFamily: styles.fontFamily,
+                    fontWeight: styles.fontWeight,
+                    textAlign: styles.textAlign,
+                    padding: styles.padding,
+                    margin: styles.margin,
+                    border: styles.border
+                };
+            }
+            results.push(data);
+        });
+        return results;
+    }, selector, opts);
+}
+/**
+ * Smart Product Scraper - Specialized scraper for e-commerce products
+ */
+export async function scrapeProducts(page, containerSelector, customFields) {
+    const defaultFields = {
+        title: '[class*="title"], [class*="name"], h2, h3',
+        price: '[class*="price"], [data-price]',
+        image: 'img',
+        link: 'a',
+        rating: '[class*="rating"], [class*="stars"]',
+        description: '[class*="desc"], p',
+        ...customFields
+    };
+    return await page.evaluate((container, fields) => {
+        const containers = document.querySelectorAll(container);
+        const products = [];
+        containers.forEach((containerEl) => {
+            const product = {};
+            // Extract title
+            const titleEl = containerEl.querySelector(fields.title);
+            if (titleEl) {
+                product.title = titleEl.innerText.trim();
+            }
+            // Extract price
+            const priceEl = containerEl.querySelector(fields.price);
+            if (priceEl) {
+                const priceText = priceEl.innerText.trim();
+                product.price = priceText;
+                // Try to extract numeric value
+                const priceMatch = priceText.match(/[\d,]+\.?\d*/);
+                if (priceMatch) {
+                    product.priceNumeric = parseFloat(priceMatch[0].replace(/,/g, ''));
+                }
+            }
+            // Extract image
+            const imageEl = containerEl.querySelector(fields.image);
+            if (imageEl) {
+                product.image = {
+                    src: imageEl.src,
+                    alt: imageEl.alt,
+                    srcset: imageEl.srcset || null
+                };
+            }
+            // Extract link
+            const linkEl = containerEl.querySelector(fields.link);
+            if (linkEl) {
+                product.url = linkEl.href;
+            }
+            // Extract rating
+            const ratingEl = containerEl.querySelector(fields.rating);
+            if (ratingEl) {
+                const ratingText = ratingEl.innerText.trim();
+                product.rating = ratingText;
+                // Try to extract numeric rating
+                const ratingMatch = ratingText.match(/(\d+\.?\d*)/);
+                if (ratingMatch) {
+                    product.ratingNumeric = parseFloat(ratingMatch[1]);
+                }
+            }
+            // Extract description
+            const descEl = containerEl.querySelector(fields.description);
+            if (descEl) {
+                product.description = descEl.innerText.trim();
+            }
+            // Only add if at least title or link exists
+            if (product.title || product.url) {
+                products.push(product);
+            }
+        });
+        return products;
+    }, containerSelector, defaultFields);
+}
+/**
+ * Smart Article Scraper - Specialized scraper for articles/blog posts
+ */
+export async function scrapeArticles(page, containerSelector, customFields) {
+    const defaultFields = {
+        title: 'h1, h2, h3, [class*="title"]',
+        author: '[class*="author"], [rel="author"]',
+        date: '[class*="date"], time',
+        content: '[class*="content"], [class*="body"], article, p',
+        image: 'img',
+        link: 'a',
+        category: '[class*="category"], [class*="tag"]',
+        ...customFields
+    };
+    return await page.evaluate((container, fields) => {
+        const containers = document.querySelectorAll(container);
+        const articles = [];
+        containers.forEach((containerEl) => {
+            const article = {};
+            // Extract each field
+            Object.entries(fields).forEach(([fieldName, selector]) => {
+                const element = containerEl.querySelector(selector);
+                if (element) {
+                    if (fieldName === 'image') {
+                        article[fieldName] = {
+                            src: element.src,
+                            alt: element.alt
+                        };
+                    }
+                    else if (fieldName === 'link') {
+                        article[fieldName] = element.href;
+                    }
+                    else if (fieldName === 'date') {
+                        article[fieldName] = element.getAttribute('datetime') ||
+                            element.innerText.trim();
+                    }
+                    else {
+                        article[fieldName] = element.innerText.trim();
+                    }
+                }
+            });
+            // Only add if at least title exists
+            if (article.title) {
+                articles.push(article);
+            }
+        });
+        return articles;
+    }, containerSelector, defaultFields);
+}