llm-search-tools 1.1.0

package/dist/modules/scraper.js

@@ -0,0 +1,640 @@
+"use strict";
+// scraper.ts - unified scraper with bot detection, proxy support, and content extraction
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.searchGoogle = searchGoogle;
+exports.searchGoogleLegacy = searchGoogle;
+exports.searchDuckDuckGo = searchDuckDuckGo;
+exports.searchDuckDuckGoLegacy = searchDuckDuckGo;
+exports.search = search;
+exports.getWebpageContent = getWebpageContent;
+exports.getWebpageText = getWebpageText;
+exports.isUrlAccessible = isUrlAccessible;
+const google_sr_1 = require("google-sr");
+const duck_duck_scrape_1 = require("duck-duck-scrape");
+const puppeteer_extra_1 = __importDefault(require("puppeteer-extra"));
+const puppeteer_extra_plugin_stealth_1 = __importDefault(require("puppeteer-extra-plugin-stealth"));
+const readability_1 = require("@mozilla/readability");
+const jsdom_1 = require("jsdom");
+const wikipedia_1 = require("./wikipedia");
+const hackernews_1 = require("./hackernews");
+// Use stealth plugin
+puppeteer_extra_1.default.use((0, puppeteer_extra_plugin_stealth_1.default)());
+const defaultOptions = {
+    limit: 10,
+    safeSearch: true,
+    timeout: 10000,
+    forcePuppeteer: false,
+    antiBot: {
+        enabled: true,
+        maxRetries: 3,
+        retryDelay: 2000
+    }
+};
+// Rate limiting parameters
+const MIN_DELAY_BETWEEN_SEARCHES = 5000; // 5 seconds for DuckDuckGo
+const GOOGLE_DELAY = 2000; // 2 seconds for Google
+const MAX_RETRIES = 3;
+const RETRY_DELAY = 2000; // 2 seconds between retries
+let lastDDGSearchTime = 0;
+let lastGoogleSearchTime = 0;
+// Cache for search results
+const searchCache = new Map();
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+// Bot detection patterns
+const BOT_PROTECTION_PATTERNS = {
+    cloudflare: [
+        'cf-ray',
+        '__cf_bm',
+        'cloudflare',
+        'challenge-platform',
+        'Just a moment...',
+        'Checking your browser',
+        'DDoS protection by Cloudflare'
+    ],
+    perimeterx: [
+        '_px',
+        'perimeterx',
+        'px-captcha',
+        'PX',
+        'bot-management'
+    ],
+    akamai: [
+        'akamai',
+        'ak_bmsc',
+        'akamaighost',
+        'akamaized',
+        'edgekey'
+    ],
+    datadome: [
+        'datadome',
+        '__ddg_',
+        'x-datadome',
+        'ddg-',
+        'bot-detection'
+    ],
+    generic: [
+        'captcha',
+        'recaptcha',
+        'hcaptcha',
+        'access denied',
+        '403 forbidden',
+        'rate limit',
+        'too many requests',
+        'blocked',
+        'security check',
+        'unauthorized'
+    ]
+};
+// Helper function to detect bot protection
+function detectBotProtection(headers, body) {
+    // Check headers
+    for (const [key, value] of headers.entries()) {
+        const headerContent = `${key}: ${value}`.toLowerCase();
+        for (const patterns of Object.values(BOT_PROTECTION_PATTERNS)) {
+            for (const pattern of patterns) {
+                if (headerContent.includes(pattern.toLowerCase())) {
+                    return true;
+                }
+            }
+        }
+    }
+    // Check body content
+    const bodyLower = body.toLowerCase();
+    for (const patterns of Object.values(BOT_PROTECTION_PATTERNS)) {
+        for (const pattern of patterns) {
+            if (bodyLower.includes(pattern.toLowerCase())) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+// Parse proxy configuration
+function parseProxyConfig(proxy) {
+    if (!proxy)
+        return null;
+    if (typeof proxy === 'string') {
+        // Parse proxy URL
+        try {
+            const url = new URL(proxy);
+            return {
+                url: proxy,
+                type: url.protocol.replace(':', '')
+            };
+        }
+        catch {
+            throw new Error('Invalid proxy URL format');
+        }
+    }
+    // Build proxy URL from config
+    const auth = proxy.auth ? `${proxy.auth.username}:${proxy.auth.password}@` : '';
+    const proxyUrl = `${proxy.type}://${auth}${proxy.host}:${proxy.port}`;
+    return {
+        url: proxyUrl,
+        type: proxy.type
+    };
+}
+// Create realistic headers for basic requests
+function createRealisticHeaders() {
+    const userAgents = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0'
+    ];
+    return {
+        'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'DNT': '1',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Cache-Control': 'max-age=0'
+    };
+}
+// Puppeteer stealth configuration with enhanced options
+async function createStealthBrowser(proxy) {
+    const launchOptions = {
+        headless: 'new',
+        args: [
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            '--disable-dev-shm-usage',
+            '--disable-accelerated-2d-canvas',
+            '--no-first-run',
+            '--no-zygote',
+            '--single-process',
+            '--disable-gpu',
+            '--disable-web-security',
+            '--disable-features=VizDisplayCompositor',
+            '--ignore-certificate-errors',
+            '--ignore-certificate-errors-spki-list'
+        ]
+    };
+    if (proxy) {
+        launchOptions.args.push(`--proxy-server=${proxy.url}`);
+    }
+    const browser = await puppeteer_extra_1.default.launch(launchOptions);
+    // Additional stealth measures
+    await browser.defaultBrowserContext().overridePermissions('https://www.google.com', []);
+    await browser.defaultBrowserContext().overridePermissions('https://duckduckgo.com', []);
+    return browser;
+}
+// Helper function to enforce rate limiting
+async function enforceRateLimit(searchType) {
+    const now = Date.now();
+    const delay = searchType === "ddg" ? MIN_DELAY_BETWEEN_SEARCHES : GOOGLE_DELAY;
+    const lastTime = searchType === "ddg" ? lastDDGSearchTime : lastGoogleSearchTime;
+    const timeSinceLastSearch = now - lastTime;
+    if (timeSinceLastSearch < delay) {
+        await new Promise((resolve) => setTimeout(resolve, delay - timeSinceLastSearch));
+    }
+    if (searchType === "ddg") {
+        lastDDGSearchTime = Date.now();
+    }
+    else {
+        lastGoogleSearchTime = Date.now();
+    }
+}
+// Helper function to get cache key
+function getCacheKey(query, options) {
+    return `${query}-${JSON.stringify(options)}`;
+}
+// Fetch with bot detection
+async function fetchWithDetection(url, options) {
+    const proxy = parseProxyConfig(options.proxy);
+    const headers = createRealisticHeaders();
+    const fetchOptions = {
+        headers,
+        timeout: options.timeout || 10000
+    };
+    if (proxy) {
+        try {
+            let agent;
+            if (proxy.type === 'socks4' || proxy.type === 'socks5') {
+                const { SocksProxyAgent } = await import('socks-proxy-agent');
+                agent = new SocksProxyAgent(proxy.url);
+            }
+            else {
+                const { HttpsProxyAgent } = await import('https-proxy-agent');
+                agent = new HttpsProxyAgent(proxy.url);
+            }
+            fetchOptions.agent = agent;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw {
+                message: `Proxy connection failed: ${errorMessage}`,
+                code: 'PROXY_CONNECTION_FAILED',
+                originalError: error
+            };
+        }
+    }
+    try {
+        const response = await fetch(url, fetchOptions);
+        const body = await response.text();
+        if (detectBotProtection(response.headers, body)) {
+            throw new Error('Bot protection detected');
+        }
+        return {
+            headers: response.headers,
+            body
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        if (errorMessage.includes('407') || errorMessage.includes('authentication')) {
+            throw {
+                message: 'Proxy authentication failed',
+                code: 'PROXY_AUTH_FAILED',
+                originalError: error
+            };
+        }
+        if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ENOTFOUND')) {
+            throw {
+                message: 'Proxy connection refused',
+                code: 'PROXY_CONNECTION_REFUSED',
+                originalError: error
+            };
+        }
+        throw error;
+    }
+}
+// Search using Puppeteer
+async function searchWithPuppeteer(query, searchType, options) {
+    const proxy = parseProxyConfig(options.proxy);
+    const browser = await createStealthBrowser(proxy || undefined);
+    const page = await browser.newPage();
+    try {
+        // Set realistic viewport
+        await page.setViewport({ width: 1920, height: 1080 });
+        // Set extra headers
+        await page.setExtraHTTPHeaders(createRealisticHeaders());
+        // Navigate to search engine
+        const searchUrl = searchType === "google"
+            ? `https://www.google.com/search?q=${encodeURIComponent(query)}`
+            : `https://duckduckgo.com/?q=${encodeURIComponent(query)}`;
+        await page.goto(searchUrl, { waitUntil: 'networkidle2' });
+        // Wait for results
+        await page.waitForSelector(searchType === "google" ? 'div.g' : '#links .result', { timeout: 10000 });
+        // Extract results
+        const results = await page.evaluate((limit) => {
+            const items = [];
+            if (window.location.hostname.includes('google')) {
+                const elements = document.querySelectorAll('div.g');
+                for (let i = 0; i < Math.min(elements.length, limit || 10); i++) {
+                    const el = elements[i];
+                    const titleEl = el.querySelector('h3');
+                    const linkEl = el.querySelector('a');
+                    const snippetEl = el.querySelector('.VwiC3b');
+                    if (titleEl && linkEl) {
+                        items.push({
+                            title: titleEl.textContent || '',
+                            url: linkEl.href || '',
+                            snippet: snippetEl?.textContent || '',
+                            source: 'google'
+                        });
+                    }
+                }
+            }
+            else {
+                const elements = document.querySelectorAll('#links .result');
+                for (let i = 0; i < Math.min(elements.length, limit || 10); i++) {
+                    const el = elements[i];
+                    const titleEl = el.querySelector('h2');
+                    const linkEl = el.querySelector('a');
+                    const snippetEl = el.querySelector('.result__snippet');
+                    if (titleEl && linkEl) {
+                        items.push({
+                            title: titleEl.textContent || '',
+                            url: linkEl.href || '',
+                            snippet: snippetEl?.textContent || '',
+                            source: 'duckduckgo'
+                        });
+                    }
+                }
+            }
+            return items;
+        }, options.limit);
+        return results;
+    }
+    finally {
+        await browser.close();
+    }
+}
+async function searchGoogle(query, options = {}) {
+    try {
+        const opts = { ...defaultOptions, ...options };
+        const cacheKey = getCacheKey(query, opts);
+        const cached = searchCache.get(cacheKey);
+        if (cached &&
+            cached.source === "google" &&
+            Date.now() - cached.timestamp < CACHE_TTL) {
+            return cached.results;
+        }
+        await enforceRateLimit("google");
+        // Try basic fetch first unless Puppeteer is forced
+        if (!opts.forcePuppeteer) {
+            try {
+                const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;
+                await fetchWithDetection(searchUrl, opts);
+                // If no bot detection, use library
+                const results = await (0, google_sr_1.search)({
+                    query,
+                    parsers: [google_sr_1.OrganicResult],
+                    noPartialResults: true,
+                    requestConfig: { queryParams: { safe: 'active' } }
+                });
+                const formattedResults = results.map((r) => ({
+                    title: r.title || "",
+                    url: r.link || "",
+                    snippet: r.description || "",
+                    source: "google",
+                }));
+                searchCache.set(cacheKey, {
+                    results: formattedResults,
+                    timestamp: Date.now(),
+                    source: "google",
+                });
+                return formattedResults;
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                if (errorMessage === 'Bot protection detected' && opts.antiBot?.enabled) {
+                    console.warn('Bot protection detected, falling back to Puppeteer...');
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        // Use Puppeteer as fallback
+        const results = await searchWithPuppeteer(query, "google", opts);
+        searchCache.set(cacheKey, {
+            results,
+            timestamp: Date.now(),
+            source: "google",
+        });
+        return results;
+    }
+    catch (error) {
+        throw {
+            message: "google search failed :(",
+            code: "GOOGLE_SEARCH_ERROR",
+            originalError: error,
+        };
+    }
+}
+async function searchDuckDuckGo(query, options = {}) {
+    const opts = { ...defaultOptions, ...options };
+    const cacheKey = getCacheKey(query, opts);
+    const cached = searchCache.get(cacheKey);
+    if (cached &&
+        cached.source === "duckduckgo" &&
+        Date.now() - cached.timestamp < CACHE_TTL) {
+        return cached.results;
+    }
+    let lastError;
+    for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+        try {
+            await enforceRateLimit("ddg");
+            // Try basic fetch first unless Puppeteer is forced
+            if (!opts.forcePuppeteer) {
+                try {
+                    const searchUrl = `https://duckduckgo.com/?q=${encodeURIComponent(query)}`;
+                    await fetchWithDetection(searchUrl, opts);
+                    // If no bot detection, use library
+                    const results = await (0, duck_duck_scrape_1.search)(query, {
+                        safeSearch: opts.safeSearch
+                            ? duck_duck_scrape_1.SafeSearchType.STRICT
+                            : duck_duck_scrape_1.SafeSearchType.OFF,
+                    });
+                    const formattedResults = results.results
+                        .slice(0, opts.limit)
+                        .map((r) => ({
+                        title: r.title,
+                        url: r.url,
+                        snippet: r.description,
+                        source: "duckduckgo",
+                    }));
+                    searchCache.set(cacheKey, {
+                        results: formattedResults,
+                        timestamp: Date.now(),
+                        source: "duckduckgo",
+                    });
+                    return formattedResults;
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    if (errorMessage === 'Bot protection detected' && opts.antiBot?.enabled) {
+                        console.warn('Bot protection detected, falling back to Puppeteer...');
+                    }
+                    else {
+                        throw error;
+                    }
+                }
+            }
+            // Use Puppeteer as fallback
+            const results = await searchWithPuppeteer(query, "duckduckgo", opts);
+            searchCache.set(cacheKey, {
+                results,
+                timestamp: Date.now(),
+                source: "duckduckgo",
+            });
+            return results;
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < MAX_RETRIES) {
+                console.warn(`DuckDuckGo search attempt ${attempt} failed, retrying in ${RETRY_DELAY / 1000} seconds...`);
+                await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY));
+            }
+        }
+    }
+    throw {
+        message: "duckduckgo search failed :/",
+        code: "DDG_SEARCH_ERROR",
+        originalError: lastError,
+    };
+}
+// Unified search that tries ddg first, then google
+async function search(query, options = {}) {
+    try {
+        return await searchDuckDuckGo(query, options);
+    }
+    catch (err) {
+        // fallback to google if ddg fails
+        console.warn("duckduckgo search failed, falling back to google...", err);
+        return await searchGoogle(query, options);
+    }
+}
+// ===== WEBPAGE CONTENT EXTRACTION =====
+// clean up text by removing excessive whitespace and making it more readable
+function cleanText(text) {
+    return text
+        .replace(/[\n\s\r]+/g, ' ')
+        .replace(/([.!?])\s+/g, '$1\n\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+// check url type and get appropriate handler
+function getUrlType(url) {
+    try {
+        const urlObj = new URL(url);
+        const hostname = urlObj.hostname;
+        if (hostname.includes('wikipedia.org')) {
+            return 'wikipedia';
+        }
+        if (hostname === 'news.ycombinator.com' && url.includes('item?id=')) {
+            return 'hackernews';
+        }
+        // list of domains that don't work well with readability
+        const unsupported = [
+            'youtube.com', 'youtu.be', 'vimeo.com',
+            'twitter.com', 'x.com', 'instagram.com',
+            'facebook.com', 'linkedin.com'
+        ];
+        if (unsupported.some(domain => hostname.includes(domain))) {
+            return 'unsupported';
+        }
+        return 'general';
+    }
+    catch {
+        return 'unsupported';
+    }
+}
+// get webpage content using readability with stealth puppeteer
+async function getWebpageContent(url, options) {
+    // Backward compatibility: if options is boolean, treat as usePuppeteer
+    if (typeof options === 'boolean') {
+        options = { usePuppeteer: options };
+    }
+    else if (!options) {
+        options = {};
+    }
+    try {
+        const urlType = getUrlType(url);
+        // handle special cases
+        if (urlType === 'wikipedia') {
+            const title = url.split('/wiki/')[1]?.replace(/_/g, ' ') || url;
+            const content = await (0, wikipedia_1.wikiGetContent)(title);
+            return {
+                title,
+                content,
+                textContent: cleanText(content),
+                length: content.length,
+                excerpt: content.slice(0, 200) + '...',
+                siteName: 'Wikipedia'
+            };
+        }
+        if (urlType === 'hackernews') {
+            const id = parseInt(url.split('id=')[1]);
+            const story = await (0, hackernews_1.getStoryById)(id);
+            const content = story.snippet || story.title || 'No content available';
+            const cleanedContent = cleanText(content);
+            return {
+                title: story.title || url,
+                content: content,
+                textContent: cleanedContent,
+                length: cleanedContent.length,
+                excerpt: cleanedContent.slice(0, 200) + (cleanedContent.length > 200 ? '...' : ''),
+                siteName: 'Hacker News'
+            };
+        }
+        if (urlType === 'unsupported') {
+            return {
+                title: url,
+                content: '',
+                textContent: 'This URL type is not supported for content extraction.',
+                length: 0,
+                excerpt: 'Content not available - URL type not supported'
+            };
+        }
+        // handle general case with readability
+        let html;
+        if (options.usePuppeteer) {
+            // Use stealth puppeteer for bot-protected sites
+            const browser = await createStealthBrowser(parseProxyConfig(options.proxy) || undefined);
+            const page = await browser.newPage();
+            await page.setViewport({ width: 1920, height: 1080 });
+            await page.setExtraHTTPHeaders(createRealisticHeaders());
+            await page.goto(url, { waitUntil: 'networkidle2' });
+            html = await page.content();
+            await browser.close();
+        }
+        else {
+            try {
+                const headers = createRealisticHeaders();
+                const fetchOptions = { headers };
+                const proxy = parseProxyConfig(options.proxy);
+                if (proxy) {
+                    if (proxy.type === 'socks4' || proxy.type === 'socks5') {
+                        const { SocksProxyAgent } = await import('socks-proxy-agent');
+                        fetchOptions.agent = new SocksProxyAgent(proxy.url);
+                    }
+                    else {
+                        const { HttpsProxyAgent } = await import('https-proxy-agent');
+                        fetchOptions.agent = new HttpsProxyAgent(proxy.url);
+                    }
+                }
+                const response = await fetch(url, fetchOptions);
+                html = await response.text();
+            }
+            catch (error) {
+                // If basic fetch fails, try with puppeteer
+                console.warn('Basic fetch failed, trying with Puppeteer...', error);
+                return await getWebpageContent(url, { ...options, usePuppeteer: true });
+            }
+        }
+        const dom = new jsdom_1.JSDOM(html, { url });
+        const reader = new readability_1.Readability(dom.window.document);
+        const article = reader.parse();
+        if (!article) {
+            return {
+                title: url,
+                content: '',
+                textContent: 'Failed to extract readable content from this page.',
+                length: 0,
+                excerpt: 'Content extraction failed'
+            };
+        }
+        const cleanedText = cleanText(article.textContent || '');
+        return {
+            title: article.title || url,
+            content: article.content || '',
+            textContent: cleanedText,
+            length: cleanedText.length,
+            excerpt: article.excerpt || undefined,
+            siteName: article.siteName || undefined
+        };
+    }
+    catch (err) {
+        throw {
+            message: 'failed to get webpage content :/',
+            code: 'WEBPAGE_ERROR',
+            originalError: err
+        };
+    }
+}
+// get just the text content
+async function getWebpageText(url, options = {}) {
+    const content = await getWebpageContent(url, options);
+    return content.textContent;
+}
+// check if url is accessible
+async function isUrlAccessible(url) {
+    try {
+        const response = await fetch(url, { method: 'HEAD' });
+        return response.ok;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/modules/scrapers/anidb.d.ts

@@ -0,0 +1,8 @@
+import { MediaResult, MediaSearchOptions } from "../../types";
+/**
+ * AniDB Scraper
+ * AniDB has strict anti-bot protection ("AntiLeech").
+ * We must use Puppeteer with Stealth plugin and respect rate limits.
+ */
+export declare function searchAniDB(query: string, options?: MediaSearchOptions): Promise<MediaResult[]>;
+export declare function getAniDBDetails(url: string, options?: MediaSearchOptions): Promise<Partial<MediaResult>>;