llm-search-tools 1.1.0

package/dist/modules/scrapers/anidb.js

@@ -0,0 +1,156 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.searchAniDB = searchAniDB;
+exports.getAniDBDetails = getAniDBDetails;
+const common_1 = require("../common");
+const jsdom_1 = require("jsdom");
+// AniDB requires stricter rate limiting to avoid bans
+// They recommend 2s delay between requests
+const MIN_DELAY = 2500;
+let lastSearchTime = 0;
+async function enforceRateLimit() {
+    const now = Date.now();
+    const timeSinceLast = now - lastSearchTime;
+    if (timeSinceLast < MIN_DELAY) {
+        await new Promise((resolve) => setTimeout(resolve, MIN_DELAY - timeSinceLast));
+    }
+    lastSearchTime = Date.now();
+}
+/**
+ * AniDB Scraper
+ * AniDB has strict anti-bot protection ("AntiLeech").
+ * We must use Puppeteer with Stealth plugin and respect rate limits.
+ */
+async function searchAniDB(query, options = {}) {
+    try {
+        await enforceRateLimit();
+        const mergedOptions = {
+            ...options,
+            // Always force puppeteer for AniDB due to protection
+            forcePuppeteer: true,
+        };
+        // AniDB search URL
+        // do=animelist performs a search
+        const searchUrl = `https://anidb.net/anime/?adb.search=${encodeURIComponent(query)}&do.update=Search&noalias=1`;
+        return await scrapeAniDBWithPuppeteer(searchUrl, mergedOptions);
+    }
+    catch (error) {
+        throw {
+            message: "AniDB search failed",
+            code: "ANIDB_SEARCH_ERROR",
+            originalError: error,
+        };
+    }
+}
+async function scrapeAniDBWithPuppeteer(url, options) {
+    const proxy = (0, common_1.parseProxyConfig)(options.proxy);
+    const browser = await (0, common_1.createStealthBrowser)(proxy || undefined);
+    try {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1920, height: 1080 });
+        await page.setExtraHTTPHeaders((0, common_1.createRealisticHeaders)());
+        // Navigate with a slightly longer timeout for AniDB
+        await page.goto(url, { waitUntil: "networkidle2", timeout: 30000 });
+        // Wait for the anime list table
+        try {
+            await page.waitForSelector("table.animelist", { timeout: 10000 });
+        }
+        catch (e) {
+            // Check if we got a single result redirect (AniDB sometimes redirects directly to the anime page)
+            const currentUrl = page.url();
+            if (currentUrl.includes("/anime/") && !currentUrl.includes("adb.search")) {
+                // Single result found
+                const html = await page.content();
+                const dom = new jsdom_1.JSDOM(html);
+                const singleResult = parseAniDBSinglePage(dom.window.document, currentUrl);
+                return singleResult ? [singleResult] : [];
+            }
+            return [];
+        }
+        const html = await page.content();
+        const dom = new jsdom_1.JSDOM(html);
+        return parseAniDBList(dom.window.document);
+    }
+    finally {
+        await browser.close();
+    }
+}
+async function getAniDBDetails(url, options = {}) {
+    try {
+        await enforceRateLimit();
+        const proxy = (0, common_1.parseProxyConfig)(options.proxy);
+        const browser = await (0, common_1.createStealthBrowser)(proxy || undefined);
+        try {
+            const page = await browser.newPage();
+            await page.setViewport({ width: 1920, height: 1080 });
+            await page.setExtraHTTPHeaders((0, common_1.createRealisticHeaders)());
+            await page.goto(url, { waitUntil: "networkidle2", timeout: 30000 });
+            const html = await page.content();
+            const dom = new jsdom_1.JSDOM(html);
+            const result = parseAniDBSinglePage(dom.window.document, url);
+            return result || {};
+        }
+        finally {
+            await browser.close();
+        }
+    }
+    catch (e) {
+        return {};
+    }
+}
+function parseAniDBList(doc) {
+    const results = [];
+    const rows = doc.querySelectorAll("table.animelist tbody tr");
+    rows.forEach((row) => {
+        if (results.length >= 10)
+            return;
+        // AniDB list columns: ID, Icon, Title, Type, Episodes, Rating, etc.
+        // The title is usually in the "name" column (class .name or depending on layout)
+        const titleLink = row.querySelector("td[data-label='Title'] a, td.name a");
+        const imgEl = row.querySelector("img"); // Often thumbnails are small or hidden on list view
+        if (titleLink) {
+            const title = titleLink.textContent?.trim() || "";
+            let href = titleLink.getAttribute("href") || "";
+            if (href && !href.startsWith("http")) {
+                href = `https://anidb.net${href}`;
+            }
+            // Try to get other metadata from columns if available
+            const typeEl = row.querySelector("td[data-label='Type']");
+            const ratingEl = row.querySelector("td[data-label='Rating']");
+            const rating = ratingEl?.textContent?.trim();
+            const type = typeEl?.textContent?.trim(); // e.g. TV, Movie, OVA
+            results.push({
+                title,
+                url: href,
+                rating,
+                description: type ? `Type: ${type}` : undefined,
+                source: "anidb",
+                mediaType: "anime",
+            });
+        }
+    });
+    return results;
+}
+function parseAniDBSinglePage(doc, url) {
+    // Parse a specific anime page if we get redirected there
+    const titleEl = doc.querySelector("h1.anime");
+    if (!titleEl)
+        return null;
+    const title = titleEl.textContent?.replace("Anime:", "").trim() || "";
+    const descriptionEl = doc.querySelector("div.desc");
+    const ratingEl = doc.querySelector("tr.rating td.value");
+    const imgEl = doc.querySelector("div.image img");
+    let posterUrl = imgEl?.getAttribute("src") || undefined;
+    if (posterUrl && !posterUrl.startsWith("http")) {
+        posterUrl = `https://anidb.net${posterUrl}`;
+    }
+    return {
+        title,
+        url,
+        description: descriptionEl?.textContent?.trim(),
+        rating: ratingEl?.textContent?.trim(),
+        posterUrl,
+        source: "anidb",
+        mediaType: "anime",
+    };
+}

package/dist/modules/scrapers/duckduckgo.d.ts

@@ -0,0 +1,6 @@
+import { ScraperOptions, SearchResult } from "../../types";
+/**
+ * Extracts the "Answer Box" or "Instant Answer" from the DuckDuckGo Search DOM
+ */
+export declare function extractAnswerBox(doc: Document): string | undefined;
+export declare function searchDuckDuckGo(query: string, options?: ScraperOptions): Promise<SearchResult[]>;

package/dist/modules/scrapers/duckduckgo.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractAnswerBox = extractAnswerBox;
+exports.searchDuckDuckGo = searchDuckDuckGo;
+const common_1 = require("../common");
+const jsdom_1 = require("jsdom");
+/**
+ * Extracts the "Answer Box" or "Instant Answer" from the DuckDuckGo Search DOM
+ */
+function extractAnswerBox(doc) {
+    // 1. Abstract / Wikipedia Snippet - .module__text
+    const abstract = doc.querySelector(".module__text");
+    if (abstract && abstract.textContent) {
+        return abstract.textContent.trim();
+    }
+    // 2. Definition / Answer - .zci__def__text
+    const definition = doc.querySelector(".zci__def__text");
+    if (definition && definition.textContent) {
+        return definition.textContent.trim();
+    }
+    // 3. Calculator / Unit Converter - .c-base__title
+    const calculator = doc.querySelector(".c-base__title");
+    if (calculator && calculator.textContent) {
+        return calculator.textContent.trim();
+    }
+    // 4. Generic Fact - .zci__body
+    const fact = doc.querySelector(".zci__body");
+    if (fact && fact.textContent) {
+        return fact.textContent.trim();
+    }
+    return undefined;
+}
+// Rate limiting parameters
+const MIN_DELAY_BETWEEN_SEARCHES = 2000;
+let lastDDGSearchTime = 0;
+// Cache for search results
+const searchCache = new Map();
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+// Helper function to enforce rate limiting
+async function enforceRateLimit() {
+    const now = Date.now();
+    const timeSinceLastSearch = now - lastDDGSearchTime;
+    if (timeSinceLastSearch < MIN_DELAY_BETWEEN_SEARCHES) {
+        await new Promise((resolve) => setTimeout(resolve, MIN_DELAY_BETWEEN_SEARCHES - timeSinceLastSearch));
+    }
+    lastDDGSearchTime = Date.now();
+}
+/**
+ * Extract the direct URL from a DuckDuckGo redirect URL
+ */
+function extractDirectUrl(duckduckgoUrl) {
+    try {
+        let urlStr = duckduckgoUrl;
+        // Handle relative URLs from DuckDuckGo
+        if (urlStr.startsWith("//")) {
+            urlStr = "https:" + urlStr;
+        }
+        else if (urlStr.startsWith("/")) {
+            urlStr = "https://duckduckgo.com" + urlStr;
+        }
+        const url = new URL(urlStr);
+        // Extract direct URL from DuckDuckGo redirect
+        if (url.hostname === "duckduckgo.com" && url.pathname === "/l/") {
+            const uddg = url.searchParams.get("uddg");
+            if (uddg) {
+                return decodeURIComponent(uddg);
+            }
+        }
+        // Handle ad redirects
+        if (url.hostname === "duckduckgo.com" && url.pathname === "/y.js") {
+            const u3 = url.searchParams.get("u3");
+            if (u3) {
+                try {
+                    const decodedU3 = decodeURIComponent(u3);
+                    const u3Url = new URL(decodedU3);
+                    const clickUrl = u3Url.searchParams.get("ld");
+                    if (clickUrl) {
+                        return decodeURIComponent(clickUrl);
+                    }
+                    return decodedU3;
+                }
+                catch {
+                    return urlStr;
+                }
+            }
+        }
+        return urlStr;
+    }
+    catch {
+        // If URL parsing fails, try to extract URL from a basic string match
+        const urlMatch = duckduckgoUrl.match(/https?:\/\/[^\s<>"]+/);
+        if (urlMatch) {
+            return urlMatch[0];
+        }
+        return duckduckgoUrl;
+    }
+}
+/**
+ * Generate a Jina AI URL for a given website URL
+ */
+function getJinaAiUrl(url) {
+    try {
+        const urlObj = new URL(url);
+        return `https://r.jina.ai/${urlObj.href}`;
+    }
+    catch {
+        return "";
+    }
+}
+// Search using Puppeteer
+async function searchWithPuppeteer(query, options) {
+    const proxy = (0, common_1.parseProxyConfig)(options.proxy);
+    const browser = await (0, common_1.createStealthBrowser)(proxy || undefined);
+    const page = await browser.newPage();
+    try {
+        // Set realistic viewport
+        await page.setViewport({ width: 1920, height: 1080 });
+        await page.setExtraHTTPHeaders((0, common_1.createRealisticHeaders)());
+        if (options.category === "images") {
+            const searchUrl = `https://duckduckgo.com/?q=${encodeURIComponent(query)}&iax=images&ia=images`;
+            await page.goto(searchUrl, { waitUntil: "networkidle2" });
+            // Wait for image results - DDG images usually loaded in tiles
+            try {
+                await page.waitForSelector(".tile--img", { timeout: 10000 });
+            }
+            catch (e) {
+                // continue
+            }
+            const results = await page.evaluate((limit) => {
+                const items = [];
+                const elements = document.querySelectorAll(".tile--img");
+                for (let i = 0; i < Math.min(elements.length, limit || 20); i++) {
+                    const el = elements[i];
+                    // Title
+                    const titleEl = el.querySelector(".tile__title");
+                    const title = titleEl?.textContent || "Image";
+                    // Source link
+                    const linkEl = el.querySelector("a.tile--img__sub");
+                    const url = linkEl?.href || "";
+                    // Thumbnail/Image
+                    const imgEl = el.querySelector("img.tile--img__img");
+                    const imageUrl = imgEl?.src || imgEl?.getAttribute("data-src") || "";
+                    if (url && imageUrl) {
+                        items.push({
+                            title,
+                            url,
+                            snippet: title,
+                            imageUrl: imageUrl,
+                            thumbnailUrl: imageUrl,
+                            source: "duckduckgo-images",
+                        });
+                    }
+                }
+                return items;
+            }, options.limit);
+            return results;
+        }
+        const searchUrl = `https://duckduckgo.com/?q=${encodeURIComponent(query)}`;
+        await page.goto(searchUrl, { waitUntil: "networkidle2" });
+        await page.waitForSelector("#links .result", { timeout: 10000 });
+        const results = await page.evaluate((limit) => {
+            const items = [];
+            const elements = document.querySelectorAll("#links .result");
+            for (let i = 0; i < Math.min(elements.length, limit || 10); i++) {
+                const el = elements[i];
+                const titleEl = el.querySelector("h2");
+                const linkEl = el.querySelector("a");
+                const snippetEl = el.querySelector(".result__snippet");
+                if (titleEl && linkEl) {
+                    items.push({
+                        title: titleEl.textContent || "",
+                        url: linkEl.href || "",
+                        snippet: snippetEl?.textContent || "",
+                        source: "duckduckgo",
+                    });
+                }
+            }
+            return items;
+        }, options.limit);
+        return results;
+    }
+    finally {
+        await browser.close();
+    }
+}
+async function searchDuckDuckGo(query, options = {}) {
+    try {
+        const mergedOptions = {
+            limit: 10,
+            safeSearch: true,
+            timeout: 10000,
+            forcePuppeteer: false,
+            antiBot: {
+                enabled: true,
+                maxRetries: 3,
+                retryDelay: 2000,
+            },
+            ...options,
+        };
+        const cacheKey = (0, common_1.getCacheKey)(query, mergedOptions);
+        const cached = searchCache.get(cacheKey);
+        if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+            return cached.results;
+        }
+        await enforceRateLimit();
+        // Try HTML scraping first unless Puppeteer is forced or we are searching for images (images require JS/Puppeteer)
+        if (!mergedOptions.forcePuppeteer && mergedOptions.category !== "images") {
+            try {
+                let searchUrl = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+                if (mergedOptions.category === "news") {
+                    searchUrl += "&iar=news&ia=news";
+                }
+                const { body } = await (0, common_1.fetchWithDetection)(searchUrl, mergedOptions);
+                const dom = new jsdom_1.JSDOM(body);
+                const doc = dom.window.document;
+                const elements = doc.querySelectorAll(".result");
+                const results = [];
+                // Use a simple loop with index to respect limit
+                for (let i = 0; i < elements.length && results.length < (mergedOptions.limit || 10); i++) {
+                    const el = elements[i];
+                    const titleEl = el.querySelector(".result__title a");
+                    const linkEl = el.querySelector(".result__url"); // Usually just display URL
+                    const snippetEl = el.querySelector(".result__snippet");
+                    // Try to extract date/source for news if available (structure might vary in HTML version)
+                    // For now, basic extraction works for both
+                    if (titleEl) {
+                        const rawLink = titleEl.getAttribute("href");
+                        const title = titleEl.textContent?.trim() || "";
+                        const snippet = snippetEl?.textContent?.trim() || "";
+                        if (rawLink && title) {
+                            const url = extractDirectUrl(rawLink);
+                            if (url && url.startsWith("http")) {
+                                results.push({
+                                    title,
+                                    url,
+                                    snippet,
+                                    source: mergedOptions.category === "news" ? "duckduckgo-news" : "duckduckgo",
+                                });
+                            }
+                        }
+                    }
+                }
+                if (results.length > 0) {
+                    searchCache.set(cacheKey, {
+                        results,
+                        timestamp: Date.now(),
+                        source: "duckduckgo",
+                    });
+                    return results;
+                }
+                // If no results found via HTML, might be blocked or empty, try puppeteer?
+                // Or if it was a genuine empty result.
+                // Let's assume if 0 results in HTML but page loaded, we might try puppeteer as backup
+                // if we suspect bot detection, but if fetchWithDetection didn't throw, maybe it's just no results.
+                // However, DDG HTML version sometimes gives 0 results for complex queries where JS version works.
+                // So fallback is good.
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                if (errorMessage === "Bot protection detected" && mergedOptions.antiBot?.enabled) {
+                    // Silent fallback
+                }
+                else {
+                    // Silent fallback
+                }
+            }
+        }
+        // Use Puppeteer as fallback
+        const results = await searchWithPuppeteer(query, mergedOptions);
+        searchCache.set(cacheKey, {
+            results,
+            timestamp: Date.now(),
+            source: "duckduckgo",
+        });
+        return results;
+    }
+    catch (error) {
+        throw {
+            message: "duckduckgo search failed :/",
+            code: "DDG_SEARCH_ERROR",
+            originalError: error,
+        };
+    }
+}

package/dist/modules/scrapers/google-news.d.ts

@@ -0,0 +1,2 @@
+import { ScraperOptions, NewsResult } from "../../types";
+export declare function searchGoogleNews(query: string, options?: ScraperOptions): Promise<NewsResult[]>;

package/dist/modules/scrapers/google-news.js

@@ -0,0 +1,60 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.searchGoogleNews = searchGoogleNews;
+const google_news_scraper_1 = __importDefault(require("google-news-scraper"));
+const common_1 = require("../common");
+// Cache for news results
+const newsCache = new Map();
+const CACHE_TTL = 30 * 60 * 1000; // 30 minutes for news
+async function searchGoogleNews(query, options = {}) {
+    try {
+        const mergedOptions = {
+            limit: 10,
+            safeSearch: true,
+            timeout: 10000,
+            ...options,
+        };
+        const cacheKey = (0, common_1.getCacheKey)(query, mergedOptions);
+        const cached = newsCache.get(cacheKey);
+        if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+            return cached.results;
+        }
+        // google-news-scraper uses Puppeteer internally
+        const articles = await (0, google_news_scraper_1.default)({
+            searchTerm: query,
+            prettyURLs: true,
+            queryVars: {
+                hl: "en-US",
+                gl: "US",
+                ceid: "US:en",
+            },
+        });
+        const results = articles.slice(0, mergedOptions.limit).map((article) => ({
+            title: article.title,
+            url: article.link,
+            snippet: article.subtitle || "",
+            source: "google-news",
+            sourceName: article.source,
+            imageUrl: article.image,
+            publishedAt: article.time,
+        }));
+        if (results.length > 0) {
+            newsCache.set(cacheKey, {
+                results,
+                timestamp: Date.now(),
+                source: "google-news",
+            });
+        }
+        return results;
+    }
+    catch (error) {
+        throw {
+            message: "google news search failed",
+            code: "GOOGLE_NEWS_SEARCH_ERROR",
+            originalError: error,
+        };
+    }
+}

package/dist/modules/scrapers/google.d.ts

@@ -0,0 +1,6 @@
+import { ScraperOptions, SearchResult } from "../../types";
+/**
+ * Extracts the "Answer Box" or "Featured Snippet" from the Google Search DOM
+ */
+export declare function extractAnswerBox(doc: Document): string | undefined;
+export declare function searchGoogle(query: string, options?: ScraperOptions): Promise<SearchResult[]>;