llm-search-tools 1.1.0

package/dist/modules/scrape.js

@@ -0,0 +1,272 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeContent = normalizeContent;
+exports.getWebpageContent = getWebpageContent;
+exports.getWebpageText = getWebpageText;
+const readability_1 = require("@mozilla/readability");
+const jsdom_1 = require("jsdom");
+const turndown_1 = __importDefault(require("turndown"));
+const wikipedia_1 = require("./wikipedia");
+const hackernews_1 = require("./hackernews");
+const google_1 = require("./scrapers/google");
+const duckduckgo_1 = require("./scrapers/duckduckgo");
+const common_1 = require("./common");
+// Configure TurndownService once for better performance
+const turndownService = new turndown_1.default({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+});
+// check url type and get appropriate handler
+function getUrlType(url) {
+    try {
+        const urlObj = new URL(url);
+        const hostname = urlObj.hostname;
+        if (hostname.includes("wikipedia.org")) {
+            return "wikipedia";
+        }
+        if (hostname === "news.ycombinator.com" && url.includes("item?id=")) {
+            return "hackernews";
+        }
+        if (hostname.includes("google.") && urlObj.pathname.startsWith("/search")) {
+            return "google-search";
+        }
+        if (hostname.includes("duckduckgo.com") && (urlObj.searchParams.has("q") || url.includes("?q="))) {
+            return "duckduckgo-search";
+        }
+        // list of domains that don't work well with readability
+        const unsupported = [
+            "youtube.com",
+            "youtu.be",
+            "vimeo.com",
+            "twitter.com",
+            "x.com",
+            "instagram.com",
+            "facebook.com",
+            "linkedin.com",
+        ];
+        if (unsupported.some((domain) => hostname.includes(domain))) {
+            return "unsupported";
+        }
+        return "general";
+    }
+    catch {
+        return "unsupported";
+    }
+}
+// Helper to extract images from DOM
+function extractImages(doc) {
+    const imageUrls = [];
+    doc.querySelectorAll("img").forEach((img) => {
+        const src = img.src;
+        // JSDOM resolves relative URLs to absolute when initialized with url option
+        if (src && src.length > 0) {
+            imageUrls.push(src);
+        }
+    });
+    return imageUrls;
+}
+// Helper to extract favicon from DOM
+function extractFavicon(doc, url) {
+    const iconLinks = doc.querySelectorAll("link[rel*='icon']");
+    if (iconLinks.length > 0) {
+        return iconLinks[0].href;
+    }
+    try {
+        return new URL("/favicon.ico", url).href;
+    }
+    catch {
+        return undefined;
+    }
+}
+function normalizeContent(params) {
+    const { url, html, title, siteName, fallbackFavicon, skipReadability } = params;
+    const dom = new jsdom_1.JSDOM(html, { url });
+    const doc = dom.window.document;
+    // Extract metadata using helpers
+    const favicon = extractFavicon(doc, url) || fallbackFavicon;
+    const imageUrls = extractImages(doc);
+    // Use Readability for better content extraction unless explicitly skipped
+    let finalContent = html;
+    let finalTitle = title || url;
+    let textContent = "";
+    let excerpt;
+    let finalSiteName = siteName;
+    if (!skipReadability) {
+        const reader = new readability_1.Readability(doc);
+        const article = reader.parse();
+        if (article) {
+            finalContent = article.content || html;
+            finalTitle = article.title || title || url;
+            textContent = (0, common_1.cleanText)(article.textContent || "");
+            excerpt = article.excerpt || undefined;
+            finalSiteName = siteName || article.siteName || undefined;
+        }
+        else {
+            // Readability failed, fall back to raw extraction
+            textContent = (0, common_1.cleanText)(doc.body?.textContent || "");
+            excerpt = textContent.slice(0, 200) + (textContent.length > 200 ? "..." : "");
+        }
+    }
+    else {
+        // Skip Readability - use raw extraction
+        textContent = (0, common_1.cleanText)(doc.body?.textContent || "");
+        excerpt = textContent.slice(0, 200) + (textContent.length > 200 ? "..." : "");
+    }
+    const markdown = turndownService.turndown(finalContent);
+    return {
+        title: finalTitle,
+        content: finalContent,
+        textContent,
+        length: textContent.length,
+        excerpt,
+        siteName: finalSiteName,
+        favicon,
+        imageUrls,
+        markdown,
+        rawHtml: html,
+    };
+}
+// get webpage content using readability with stealth puppeteer
+async function getWebpageContent(url, options) {
+    // Backward compatibility: if options is boolean, treat as usePuppeteer
+    if (typeof options === "boolean") {
+        options = { usePuppeteer: options };
+    }
+    else if (!options) {
+        options = {};
+    }
+    // Cast options to the correct type for internal use
+    const opts = options;
+    try {
+        const urlType = getUrlType(url);
+        // handle special cases
+        if (urlType === "wikipedia") {
+            const title = url.split("/wiki/")[1]?.replace(/_/g, " ") || url;
+            const html = await (0, wikipedia_1.wikiGetContent)(title);
+            return normalizeContent({
+                url,
+                html,
+                title,
+                siteName: "Wikipedia",
+                fallbackFavicon: "https://en.wikipedia.org/static/favicon/wikipedia.ico",
+                skipReadability: true, // Wikipedia content is already clean from their API
+            });
+        }
+        if (urlType === "hackernews") {
+            const idStr = url.split("id=")[1];
+            const id = parseInt(idStr);
+            const story = await (0, hackernews_1.getStoryById)(id);
+            const html = story.snippet || story.title || "No content available";
+            return normalizeContent({
+                url,
+                html,
+                title: story.title || url,
+                siteName: "Hacker News",
+                fallbackFavicon: "https://news.ycombinator.com/favicon.ico",
+                skipReadability: true, // HN snippets are already clean
+            });
+        }
+        if (urlType === "unsupported") {
+            return normalizeContent({
+                url,
+                html: "<html><body><p>This URL type is not supported for content extraction.</p></body></html>",
+                title: url,
+                skipReadability: true,
+            });
+        }
+        // handle general case with readability
+        let html;
+        if (opts.usePuppeteer) {
+            // Use stealth puppeteer for bot-protected sites
+            const proxy = (0, common_1.parseProxyConfig)(opts.proxy);
+            const browser = await (0, common_1.createStealthBrowser)(proxy || undefined);
+            try {
+                const pages = await browser.pages();
+                const page = pages.length > 0 ? pages[0] : await browser.newPage();
+                await page.setViewport({ width: 1920, height: 1080 });
+                await page.setExtraHTTPHeaders((0, common_1.createRealisticHeaders)());
+                await page.goto(url, { waitUntil: "networkidle2" });
+                html = await page.content();
+            }
+            finally {
+                await browser.close();
+            }
+        }
+        else {
+            try {
+                const headers = (0, common_1.createRealisticHeaders)();
+                const fetchOptions = { headers };
+                const proxy = (0, common_1.parseProxyConfig)(opts.proxy);
+                if (proxy) {
+                    if (proxy.type === "socks4" || proxy.type === "socks5") {
+                        const { SocksProxyAgent } = await import("socks-proxy-agent");
+                        fetchOptions.agent = new SocksProxyAgent(proxy.url);
+                    }
+                    else {
+                        const { HttpsProxyAgent } = await import("https-proxy-agent");
+                        fetchOptions.agent = new HttpsProxyAgent(proxy.url);
+                    }
+                }
+                const response = await fetch(url, fetchOptions);
+                html = await response.text();
+                if ((0, common_1.detectBotProtection)(response.headers, html)) {
+                    // If bot protection detected, re-run with puppeteer
+                    return await getWebpageContent(url, { ...opts, usePuppeteer: true });
+                }
+            }
+            catch {
+                // If basic fetch fails, try with puppeteer
+                return await getWebpageContent(url, { ...opts, usePuppeteer: true });
+            }
+        }
+        // Extract Answer Box for Search Engines
+        if (urlType === "google-search") {
+            const dom = new jsdom_1.JSDOM(html);
+            const answer = (0, google_1.extractAnswerBox)(dom.window.document);
+            if (answer) {
+                return normalizeContent({
+                    url,
+                    html: `<div class="answer-box"><h1>Google Answer</h1><p>${answer}</p></div>`,
+                    title: "Google Answer",
+                    siteName: "Google",
+                    skipReadability: true,
+                    fallbackFavicon: "https://www.google.com/favicon.ico",
+                });
+            }
+        }
+        if (urlType === "duckduckgo-search") {
+            const dom = new jsdom_1.JSDOM(html);
+            const answer = (0, duckduckgo_1.extractAnswerBox)(dom.window.document);
+            if (answer) {
+                return normalizeContent({
+                    url,
+                    html: `<div class="answer-box"><h1>DuckDuckGo Answer</h1><p>${answer}</p></div>`,
+                    title: "DuckDuckGo Answer",
+                    siteName: "DuckDuckGo",
+                    skipReadability: true,
+                    fallbackFavicon: "https://duckduckgo.com/favicon.ico",
+                });
+            }
+        }
+        return normalizeContent({
+            url,
+            html,
+            skipReadability: false,
+        });
+    }
+    catch (err) {
+        throw {
+            message: "failed to get webpage content :/",
+            code: "WEBPAGE_ERROR",
+            originalError: err,
+        };
+    }
+}
+// get just the text content
+async function getWebpageText(url, options = {}) {
+    const content = await getWebpageContent(url, options);
+    return content.textContent;
+}

package/dist/modules/scrape.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/modules/scrape.test.js

@@ -0,0 +1,232 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const scrape_1 = require("./scrape");
+const common = __importStar(require("./common"));
+const wikipedia_1 = require("./wikipedia");
+const hackernews_1 = require("./hackernews");
+const readability_1 = require("@mozilla/readability");
+const jsdom_1 = require("jsdom");
+const google_1 = require("./scrapers/google");
+const duckduckgo_1 = require("./scrapers/duckduckgo");
+// Mock dependencies
+vitest_1.vi.mock("./wikipedia");
+vitest_1.vi.mock("./hackernews");
+vitest_1.vi.mock("./scrapers/google", () => ({
+    extractAnswerBox: vitest_1.vi.fn(),
+}));
+vitest_1.vi.mock("./scrapers/duckduckgo", () => ({
+    extractAnswerBox: vitest_1.vi.fn(),
+}));
+vitest_1.vi.mock("@mozilla/readability");
+vitest_1.vi.mock("jsdom");
+vitest_1.vi.mock("./common", async () => {
+    const actual = await vitest_1.vi.importActual("./common");
+    return {
+        ...actual,
+        createStealthBrowser: vitest_1.vi.fn(),
+        createRealisticHeaders: vitest_1.vi.fn().mockReturnValue({ "User-Agent": "test-agent" }),
+        parseProxyConfig: vitest_1.vi.fn(),
+        detectBotProtection: vitest_1.vi.fn().mockReturnValue(false),
+        cleanText: vitest_1.vi.fn((text) => text?.trim() || ""),
+    };
+});
+// Global fetch mock
+global.fetch = vitest_1.vi.fn();
+(0, vitest_1.describe)("Scrape Module", () => {
+    (0, vitest_1.beforeEach)(() => {
+        vitest_1.vi.resetAllMocks();
+        // Default JSDOM/Readability mocks
+        jsdom_1.JSDOM.mockImplementation(() => ({
+            window: {
+                document: {
+                    querySelectorAll: vitest_1.vi.fn().mockImplementation((selector) => {
+                        if (selector === "link[rel*='icon']") {
+                            return [{ href: "https://example.com/favicon.ico" }];
+                        }
+                        if (selector === "img") {
+                            return [{ src: "https://example.com/image1.jpg" }, { src: "https://example.com/image2.png" }];
+                        }
+                        return [];
+                    }),
+                    body: {
+                        textContent: "Test content from body",
+                    },
+                },
+            },
+        }));
+        readability_1.Readability.mockImplementation(() => ({
+            parse: vitest_1.vi.fn().mockReturnValue({
+                title: "Test Article",
+                content: "<div>Test content</div>",
+                textContent: "Test content",
+                excerpt: "Test excerpt",
+                siteName: "Test Site",
+            }),
+        }));
+    });
+    (0, vitest_1.describe)("URL Type Routing", () => {
+        (0, vitest_1.it)("should route wikipedia URLs to wiki handler", async () => {
+            wikipedia_1.wikiGetContent.mockResolvedValue("Wiki content");
+            const result = await (0, scrape_1.getWebpageContent)("https://en.wikipedia.org/wiki/Test");
+            (0, vitest_1.expect)(result.siteName).toBe("Wikipedia");
+            (0, vitest_1.expect)(result.content).toBe("Wiki content");
+            (0, vitest_1.expect)(result.favicon).toBeDefined();
+            (0, vitest_1.expect)(result.markdown).toBeDefined();
+            (0, vitest_1.expect)(wikipedia_1.wikiGetContent).toHaveBeenCalledWith("Test");
+        });
+        (0, vitest_1.it)("should route hackernews URLs to HN handler", async () => {
+            hackernews_1.getStoryById.mockResolvedValue({
+                title: "HN Story",
+                snippet: "HN Content",
+            });
+            const result = await (0, scrape_1.getWebpageContent)("https://news.ycombinator.com/item?id=12345");
+            (0, vitest_1.expect)(result.siteName).toBe("Hacker News");
+            (0, vitest_1.expect)(result.title).toBe("HN Story");
+            (0, vitest_1.expect)(result.favicon).toBeDefined();
+            (0, vitest_1.expect)(result.markdown).toBeDefined();
+            (0, vitest_1.expect)(hackernews_1.getStoryById).toHaveBeenCalledWith(12345);
+        });
+        (0, vitest_1.it)("should handle unsupported domains", async () => {
+            const result = await (0, scrape_1.getWebpageContent)("https://youtube.com/watch?v=123");
+            // Unsupported URLs now go through normalizeContent, which uses the JSDOM mock
+            (0, vitest_1.expect)(result.textContent).toBe("Test content from body");
+            (0, vitest_1.expect)(result.title).toBe("https://youtube.com/watch?v=123");
+        });
+        (0, vitest_1.it)("should route google search URLs and extract answer box", async () => {
+            // Mock fetch response
+            global.fetch.mockResolvedValue({
+                text: () => Promise.resolve("<html><body>Google Search Result</body></html>"),
+                headers: new Map(),
+            });
+            // Mock extractor to return an answer
+            google_1.extractAnswerBox.mockReturnValue("The Answer is 42");
+            const result = await (0, scrape_1.getWebpageContent)("https://www.google.com/search?q=answer");
+            (0, vitest_1.expect)(result.siteName).toBe("Google");
+            (0, vitest_1.expect)(result.title).toBe("Google Answer");
+            (0, vitest_1.expect)(result.rawHtml).toContain("The Answer is 42");
+            (0, vitest_1.expect)(google_1.extractAnswerBox).toHaveBeenCalled();
+        });
+        (0, vitest_1.it)("should route duckduckgo search URLs and extract answer box", async () => {
+            // Mock fetch response
+            global.fetch.mockResolvedValue({
+                text: () => Promise.resolve("<html><body>DDG Search Result</body></html>"),
+                headers: new Map(),
+            });
+            // Mock extractor to return an answer
+            duckduckgo_1.extractAnswerBox.mockReturnValue("The DDG Answer");
+            const result = await (0, scrape_1.getWebpageContent)("https://duckduckgo.com/?q=answer");
+            (0, vitest_1.expect)(result.siteName).toBe("DuckDuckGo");
+            (0, vitest_1.expect)(result.title).toBe("DuckDuckGo Answer");
+            (0, vitest_1.expect)(result.rawHtml).toContain("The DDG Answer");
+            (0, vitest_1.expect)(duckduckgo_1.extractAnswerBox).toHaveBeenCalled();
+        });
+    });
+    (0, vitest_1.describe)("General Scraping (Fetch)", () => {
+        (0, vitest_1.it)("should scrape content using fetch and readability", async () => {
+            global.fetch.mockResolvedValue({
+                text: () => Promise.resolve("<html><body>Test</body></html>"),
+                headers: new Map(),
+            });
+            const result = await (0, scrape_1.getWebpageContent)("https://example.com");
+            (0, vitest_1.expect)(result.title).toBe("Test Article");
+            (0, vitest_1.expect)(result.textContent).toBe("Test content");
+            (0, vitest_1.expect)(result.favicon).toBe("https://example.com/favicon.ico");
+            (0, vitest_1.expect)(result.imageUrls).toBeDefined();
+            (0, vitest_1.expect)(result.imageUrls).toContain("https://example.com/image1.jpg");
+            (0, vitest_1.expect)(result.imageUrls).toContain("https://example.com/image2.png");
+            (0, vitest_1.expect)(result.markdown).toBeDefined();
+            (0, vitest_1.expect)(result.markdown?.length).toBeGreaterThan(0);
+            (0, vitest_1.expect)(result.rawHtml).toContain("<html><body>Test</body></html>");
+            (0, vitest_1.expect)(global.fetch).toHaveBeenCalled();
+            (0, vitest_1.expect)(readability_1.Readability).toHaveBeenCalled();
+        });
+        (0, vitest_1.it)("should fallback to puppeteer if bot protection detected", async () => {
+            // First call detects bot
+            global.fetch.mockResolvedValue({
+                text: () => Promise.resolve("<html><body>Captcha</body></html>"),
+                headers: new Map(),
+            });
+            common.detectBotProtection.mockReturnValueOnce(true);
+            // Mock puppeteer part
+            const mockPage = {
+                setViewport: vitest_1.vi.fn(),
+                setExtraHTTPHeaders: vitest_1.vi.fn(),
+                goto: vitest_1.vi.fn(),
+                content: vitest_1.vi.fn().mockResolvedValue("<html><body>Real Content</body></html>"),
+            };
+            const mockBrowser = {
+                pages: vitest_1.vi.fn().mockResolvedValue([]),
+                newPage: vitest_1.vi.fn().mockResolvedValue(mockPage),
+                close: vitest_1.vi.fn(),
+            };
+            common.createStealthBrowser.mockResolvedValue(mockBrowser);
+            const result = await (0, scrape_1.getWebpageContent)("https://protected.com");
+            (0, vitest_1.expect)(common.createStealthBrowser).toHaveBeenCalled();
+            (0, vitest_1.expect)(result.title).toBe("Test Article");
+        });
+    });
+    (0, vitest_1.describe)("General Scraping (Puppeteer)", () => {
+        (0, vitest_1.it)("should use puppeteer when forced", async () => {
+            const mockPage = {
+                setViewport: vitest_1.vi.fn(),
+                setExtraHTTPHeaders: vitest_1.vi.fn(),
+                goto: vitest_1.vi.fn(),
+                content: vitest_1.vi.fn().mockResolvedValue("<html><body>Puppeteer Content</body></html>"),
+            };
+            const mockBrowser = {
+                pages: vitest_1.vi.fn().mockResolvedValue([]),
+                newPage: vitest_1.vi.fn().mockResolvedValue(mockPage),
+                close: vitest_1.vi.fn(),
+            };
+            common.createStealthBrowser.mockResolvedValue(mockBrowser);
+            await (0, scrape_1.getWebpageContent)("https://example.com", { usePuppeteer: true });
+            (0, vitest_1.expect)(global.fetch).not.toHaveBeenCalled();
+            (0, vitest_1.expect)(common.createStealthBrowser).toHaveBeenCalled();
+            (0, vitest_1.expect)(mockPage.goto).toHaveBeenCalledWith("https://example.com", vitest_1.expect.any(Object));
+        });
+    });
+    (0, vitest_1.describe)("getWebpageText", () => {
+        (0, vitest_1.it)("should return only text content", async () => {
+            global.fetch.mockResolvedValue({
+                text: () => Promise.resolve("html"),
+                headers: new Map(),
+            });
+            const text = await (0, scrape_1.getWebpageText)("https://example.com");
+            (0, vitest_1.expect)(text).toBe("Test content");
+        });
+    });
+});

package/dist/modules/scraper.d.ts

@@ -0,0 +1,12 @@
+import { ScraperOptions, SearchResult, WebpageContent } from "../types";
+export declare function searchGoogle(query: string, options?: ScraperOptions): Promise<SearchResult[]>;
+export declare function searchDuckDuckGo(query: string, options?: ScraperOptions): Promise<SearchResult[]>;
+export declare function search(query: string, options?: ScraperOptions): Promise<SearchResult[]>;
+export declare function getWebpageContent(url: string, options?: {
+    usePuppeteer?: boolean;
+} & ScraperOptions | boolean): Promise<WebpageContent>;
+export declare function getWebpageText(url: string, options?: {
+    usePuppeteer?: boolean;
+} & ScraperOptions): Promise<string>;
+export declare function isUrlAccessible(url: string): Promise<boolean>;
+export { searchGoogle as searchGoogleLegacy, searchDuckDuckGo as searchDuckDuckGoLegacy };