llm-search-tools 1.1.0

package/dist/modules/autocomplete.d.ts

@@ -0,0 +1,11 @@
+import { AutocompleteOptions, AutocompleteResult } from "../types";
+type Provider = "google" | "duckduckgo" | "yahoo" | "brave" | "yandex" | "ecosia" | "startpage" | "qwant" | "swisscows";
+/**
+ * Get autocomplete suggestions for a query
+ * @param query The search query
+ * @param provider The search provider to use (default: duckduckgo)
+ * @param options Options for the request
+ * @returns Promise<AutocompleteResult>
+ */
+export declare function getSuggestions(query: string, provider?: Provider, options?: AutocompleteOptions): Promise<AutocompleteResult>;
+export {};

package/dist/modules/autocomplete.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSuggestions = getSuggestions;
+const common_1 = require("./common");
+/**
+ * Get autocomplete suggestions for a query
+ * @param query The search query
+ * @param provider The search provider to use (default: duckduckgo)
+ * @param options Options for the request
+ * @returns Promise<AutocompleteResult>
+ */
+async function getSuggestions(query, provider = "duckduckgo", options = {}) {
+    try {
+        switch (provider) {
+            case "google":
+                return await getGoogleSuggestions(query, options);
+            case "duckduckgo":
+                return await getDuckDuckGoSuggestions(query, options);
+            case "yahoo":
+                return await getYahooSuggestions(query, options);
+            case "brave":
+                return await getBraveSuggestions(query, options);
+            case "yandex":
+                return await getYandexSuggestions(query, options);
+            case "ecosia":
+                return await getEcosiaSuggestions(query, options);
+            case "startpage":
+                return await getStartpageSuggestions(query, options);
+            case "qwant":
+                return await getQwantSuggestions(query, options);
+            case "swisscows":
+                return await getSwisscowsSuggestions(query, options);
+            default:
+                return await getDuckDuckGoSuggestions(query, options);
+        }
+    }
+    catch (error) {
+        throw {
+            message: `Failed to get suggestions from ${provider}: ${error.message}`,
+            code: "AUTOCOMPLETE_ERROR",
+            originalError: error,
+        };
+    }
+}
+async function fetchJson(url, options) {
+    // Use fetchWithDetection to handle proxies and headers
+    const { body } = await (0, common_1.fetchWithDetection)(url, {
+        timeout: options.timeout,
+        proxy: options.proxy,
+    });
+    return JSON.parse(body);
+}
+// Google
+async function getGoogleSuggestions(query, options) {
+    const url = `http://suggestqueries.google.com/complete/search?client=firefox&q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: ["query", ["sugg1", "sugg2", ...]]
+    return {
+        query,
+        suggestions: (data[1] || []).slice(0, options.limit),
+        source: "google",
+    };
+}
+// DuckDuckGo
+async function getDuckDuckGoSuggestions(query, options) {
+    const url = `https://duckduckgo.com/ac/?kl=wt-wt&q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: [{"phrase": "sugg1"}, {"phrase": "sugg2"}, ...]
+    const suggestions = data.map((item) => item.phrase).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "duckduckgo",
+    };
+}
+// Yahoo
+async function getYahooSuggestions(query, options) {
+    const url = `https://search.yahoo.com/sugg/gossip/gossip-us-fastbreak?output=sd1&command=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: {"gossip": {"results": [{"key": "sugg1"}, ...]}}
+    const suggestions = (data.gossip?.results || []).map((item) => item.key).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "yahoo",
+    };
+}
+// Brave
+async function getBraveSuggestions(query, options) {
+    const url = `https://search.brave.com/api/suggest?rich=true&source=web&country=us&q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: [ ["query", ...], ["sugg1", "sugg2", ...] ] (OpenSearch compatible-ish)
+    const suggestions = (data[1] || []).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "brave",
+    };
+}
+// Yandex
+async function getYandexSuggestions(query, options) {
+    const url = `https://yandex.com/suggest/suggest-ya.cgi?srv=morda_com_desktop&wiz=TrWth&uil=en&fact=1&v=4&icon=1&part=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: ["query", ["sugg1", "sugg2", ...]]
+    const suggestions = (data[1] || []).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "yandex",
+    };
+}
+// Ecosia
+async function getEcosiaSuggestions(query, options) {
+    const url = `https://ac.ecosia.org/?q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: {"suggestions": ["sugg1", "sugg2", ...]}
+    const suggestions = (data.suggestions || []).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "ecosia",
+    };
+}
+// Startpage
+async function getStartpageSuggestions(query, options) {
+    const url = `https://www.startpage.com/suggestions?q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: {"suggestions": [{"text": "sugg1"}, ...]}
+    const suggestions = (data.suggestions || []).map((item) => item.text).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "startpage",
+    };
+}
+// Qwant
+async function getQwantSuggestions(query, options) {
+    const url = `https://api.qwant.com/v3/suggest?q=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: {"data": {"items": [{"value": "sugg1"}, ...]}}
+    const suggestions = (data.data?.items || []).map((item) => item.value).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "qwant",
+    };
+}
+// Swisscows
+async function getSwisscowsSuggestions(query, options) {
+    const url = `https://api.swisscows.com/suggest?locale=en-US&itemsCount=${options.limit || 10}&query=${encodeURIComponent(query)}`;
+    const data = await fetchJson(url, options);
+    // Format: ["sugg1", "sugg2", ...]
+    const suggestions = (Array.isArray(data) ? data : []).slice(0, options.limit);
+    return {
+        query,
+        suggestions,
+        source: "swisscows",
+    };
+}

package/dist/modules/autocomplete.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/modules/autocomplete.test.js

@@ -0,0 +1,188 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const autocomplete_1 = require("./autocomplete");
+const common = __importStar(require("./common"));
+// Mock the common module
+vitest_1.vi.mock("./common", async () => {
+    const actual = await vitest_1.vi.importActual("./common");
+    return {
+        ...actual,
+        fetchWithDetection: vitest_1.vi.fn(),
+    };
+});
+(0, vitest_1.describe)("Autocomplete Module", () => {
+    const mockQuery = "test";
+    const mockOptions = { limit: 5 };
+    (0, vitest_1.beforeEach)(() => {
+        vitest_1.vi.clearAllMocks();
+    });
+    (0, vitest_1.it)("should get suggestions from Google", async () => {
+        const mockResponse = JSON.stringify([
+            "test",
+            ["test speed", "test internet", "test microphone"],
+            [],
+            { google: { client: "firefox" } },
+        ]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "google", mockOptions);
+        (0, vitest_1.expect)(common.fetchWithDetection).toHaveBeenCalledWith(vitest_1.expect.stringContaining("google.com"), vitest_1.expect.any(Object));
+        (0, vitest_1.expect)(result.source).toBe("google");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet", "test microphone"]);
+        (0, vitest_1.expect)(result.suggestions.length).toBeLessThanOrEqual(mockOptions.limit);
+    });
+    (0, vitest_1.it)("should get suggestions from DuckDuckGo", async () => {
+        const mockResponse = JSON.stringify([{ phrase: "test speed" }, { phrase: "test internet" }, { phrase: "tester" }]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "duckduckgo", mockOptions);
+        (0, vitest_1.expect)(common.fetchWithDetection).toHaveBeenCalledWith(vitest_1.expect.stringContaining("duckduckgo.com"), vitest_1.expect.any(Object));
+        (0, vitest_1.expect)(result.source).toBe("duckduckgo");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet", "tester"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Yahoo", async () => {
+        const mockResponse = JSON.stringify({
+            gossip: {
+                results: [{ key: "test speed" }, { key: "test internet" }],
+            },
+        });
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "yahoo", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("yahoo");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Brave", async () => {
+        // OpenSearch format
+        const mockResponse = JSON.stringify(["test", ["test speed", "test internet"]]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "brave", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("brave");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Yandex", async () => {
+        const mockResponse = JSON.stringify(["test", ["test speed", "test internet"]]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "yandex", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("yandex");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Ecosia", async () => {
+        const mockResponse = JSON.stringify({
+            suggestions: ["test speed", "test internet"],
+        });
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "ecosia", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("ecosia");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Startpage", async () => {
+        const mockResponse = JSON.stringify({
+            suggestions: [{ text: "test speed" }, { text: "test internet" }],
+        });
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "startpage", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("startpage");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Qwant", async () => {
+        const mockResponse = JSON.stringify({
+            data: {
+                items: [{ value: "test speed" }, { value: "test internet" }],
+            },
+        });
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "qwant", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("qwant");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should get suggestions from Swisscows", async () => {
+        const mockResponse = JSON.stringify(["test speed", "test internet"]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "swisscows", mockOptions);
+        (0, vitest_1.expect)(result.source).toBe("swisscows");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test speed", "test internet"]);
+    });
+    (0, vitest_1.it)("should fallback to DuckDuckGo when no provider specified", async () => {
+        const mockResponse = JSON.stringify([{ phrase: "test fallback" }]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery);
+        (0, vitest_1.expect)(result.source).toBe("duckduckgo");
+        (0, vitest_1.expect)(result.suggestions).toEqual(["test fallback"]);
+    });
+    (0, vitest_1.it)("should handle errors gracefully", async () => {
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockRejectedValue(new Error("Network Error"));
+        await (0, vitest_1.expect)((0, autocomplete_1.getSuggestions)(mockQuery, "google")).rejects.toThrow("Failed to get suggestions from google");
+    });
+    (0, vitest_1.it)("should respect limit option", async () => {
+        const mockResponse = JSON.stringify(["test", ["1", "2", "3", "4", "5", "6"]]);
+        vitest_1.vi.spyOn(common, "fetchWithDetection").mockResolvedValue({
+            body: mockResponse,
+            headers: new Headers(),
+        });
+        const result = await (0, autocomplete_1.getSuggestions)(mockQuery, "google", { limit: 3 });
+        (0, vitest_1.expect)(result.suggestions).toHaveLength(3);
+        (0, vitest_1.expect)(result.suggestions).toEqual(["1", "2", "3"]);
+    });
+});

package/dist/modules/common.d.ts

@@ -0,0 +1,26 @@
+import { ProxyConfig, ScraperOptions } from "../types";
+import type { Browser } from "puppeteer";
+export declare const BOT_PROTECTION_PATTERNS: {
+    cloudflare: string[];
+    perimeterx: string[];
+    akamai: string[];
+    datadome: string[];
+    generic: string[];
+};
+export declare function detectBotProtection(headers: Headers | Map<string, string> | Record<string, string> | object | undefined, body: string): boolean;
+export declare function parseProxyConfig(proxy?: ProxyConfig | string): {
+    url: string;
+    type: string;
+} | null;
+export declare function createRealisticHeaders(): Record<string, string>;
+export declare function createStealthBrowser(proxy?: {
+    url: string;
+    type: string;
+}): Promise<Browser>;
+export declare function fetchWithDetection(url: string, options: ScraperOptions): Promise<{
+    headers: Headers;
+    body: string;
+}>;
+export declare function isUrlAccessible(url: string): Promise<boolean>;
+export declare function cleanText(text: string): string;
+export declare function getCacheKey(query: string, options: ScraperOptions): string;

package/dist/modules/common.js

@@ -0,0 +1,263 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BOT_PROTECTION_PATTERNS = void 0;
+exports.detectBotProtection = detectBotProtection;
+exports.parseProxyConfig = parseProxyConfig;
+exports.createRealisticHeaders = createRealisticHeaders;
+exports.createStealthBrowser = createStealthBrowser;
+exports.fetchWithDetection = fetchWithDetection;
+exports.isUrlAccessible = isUrlAccessible;
+exports.cleanText = cleanText;
+exports.getCacheKey = getCacheKey;
+const puppeteer_extra_1 = __importDefault(require("puppeteer-extra"));
+const puppeteer_extra_plugin_stealth_1 = __importDefault(require("puppeteer-extra-plugin-stealth"));
+// Use stealth plugin
+puppeteer_extra_1.default.use((0, puppeteer_extra_plugin_stealth_1.default)());
+// Type guard for objects with entries method (Headers, Map, etc.)
+function hasEntries(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        "entries" in value &&
+        typeof value.entries === "function");
+}
+// Bot detection patterns
+exports.BOT_PROTECTION_PATTERNS = {
+    cloudflare: [
+        "cf-ray",
+        "__cf_bm",
+        "cloudflare",
+        "challenge-platform",
+        "Just a moment...",
+        "Checking your browser",
+        "DDoS protection by Cloudflare",
+    ],
+    perimeterx: ["_px", "perimeterx", "px-captcha", "PX", "bot-management"],
+    akamai: ["akamai", "ak_bmsc", "akamaighost", "akamaized", "edgekey"],
+    datadome: ["datadome", "__ddg_", "x-datadome", "ddg-", "bot-detection"],
+    generic: [
+        "captcha",
+        "recaptcha",
+        "hcaptcha",
+        "access denied",
+        "403 forbidden",
+        "rate limit",
+        "too many requests",
+        "blocked",
+        "security check",
+        "unauthorized",
+    ],
+};
+// Helper function to detect bot protection
+function detectBotProtection(headers, body) {
+    // Check headers
+    if (headers) {
+        // Handle Headers object or Map
+        if (hasEntries(headers)) {
+            for (const [key, value] of headers.entries()) {
+                const headerContent = `${key}: ${value}`.toLowerCase();
+                for (const patterns of Object.values(exports.BOT_PROTECTION_PATTERNS)) {
+                    for (const pattern of patterns) {
+                        if (headerContent.includes(pattern.toLowerCase())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+        // Handle plain object
+        else if (typeof headers === "object") {
+            for (const [key, value] of Object.entries(headers)) {
+                const headerContent = `${key}: ${value}`.toLowerCase();
+                for (const patterns of Object.values(exports.BOT_PROTECTION_PATTERNS)) {
+                    for (const pattern of patterns) {
+                        if (headerContent.includes(pattern.toLowerCase())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // Check body content
+    if (body) {
+        const bodyLower = body.toLowerCase();
+        for (const patterns of Object.values(exports.BOT_PROTECTION_PATTERNS)) {
+            for (const pattern of patterns) {
+                if (bodyLower.includes(pattern.toLowerCase())) {
+                    return true;
+                }
+            }
+        }
+    }
+    return false;
+}
+// Parse proxy configuration
+function parseProxyConfig(proxy) {
+    if (!proxy)
+        return null;
+    if (typeof proxy === "string") {
+        // Parse proxy URL
+        try {
+            const url = new URL(proxy);
+            return {
+                url: proxy,
+                type: url.protocol.replace(":", ""),
+            };
+        }
+        catch {
+            throw new Error("Invalid proxy URL format");
+        }
+    }
+    // Build proxy URL from config
+    const auth = proxy.auth ? `${proxy.auth.username}:${proxy.auth.password}@` : "";
+    const proxyUrl = `${proxy.type}://${auth}${proxy.host}:${proxy.port}`;
+    return {
+        url: proxyUrl,
+        type: proxy.type,
+    };
+}
+// Create realistic headers for basic requests
+function createRealisticHeaders() {
+    const userAgents = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
+    ];
+    return {
+        "User-Agent": userAgents[Math.floor(Math.random() * userAgents.length)],
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br",
+        DNT: "1",
+        Connection: "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Cache-Control": "max-age=0",
+    };
+}
+// Puppeteer stealth configuration with enhanced options
+async function createStealthBrowser(proxy) {
+    const launchOptions = {
+        headless: true,
+        args: [
+            "--no-sandbox",
+            "--disable-setuid-sandbox",
+            "--disable-dev-shm-usage",
+            "--disable-accelerated-2d-canvas",
+            "--no-first-run",
+            "--no-zygote",
+            "--single-process",
+            "--disable-gpu",
+            "--disable-web-security",
+            "--disable-features=VizDisplayCompositor",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+        ],
+    };
+    if (proxy) {
+        launchOptions.args.push(`--proxy-server=${proxy.url}`);
+    }
+    const browser = await puppeteer_extra_1.default.launch(launchOptions);
+    // Additional stealth measures
+    try {
+        const pages = await browser.pages();
+        const page = pages.length > 0 ? pages[0] : await browser.newPage();
+        const context = page.browserContext();
+        await context.overridePermissions("https://www.google.com", []);
+        await context.overridePermissions("https://duckduckgo.com", []);
+        if (pages.length === 0)
+            await page.close();
+    }
+    catch {
+        // Ignore permissions errors if context doesn't support it
+    }
+    return browser;
+}
+// Fetch with bot detection
+async function fetchWithDetection(url, options) {
+    const proxy = parseProxyConfig(options.proxy);
+    const headers = createRealisticHeaders();
+    const fetchOptions = {
+        headers,
+        timeout: options.timeout || 10000,
+    };
+    if (proxy) {
+        try {
+            let agent;
+            if (proxy.type === "socks4" || proxy.type === "socks5") {
+                const { SocksProxyAgent } = await import("socks-proxy-agent");
+                agent = new SocksProxyAgent(proxy.url);
+            }
+            else {
+                const { HttpsProxyAgent } = await import("https-proxy-agent");
+                agent = new HttpsProxyAgent(proxy.url);
+            }
+            fetchOptions.agent = agent;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw {
+                message: `Proxy connection failed: ${errorMessage}`,
+                code: "PROXY_CONNECTION_FAILED",
+                originalError: error,
+            };
+        }
+    }
+    try {
+        const response = await fetch(url, fetchOptions);
+        const body = await response.text();
+        if (detectBotProtection(response.headers, body)) {
+            throw new Error("Bot protection detected");
+        }
+        return {
+            headers: response.headers,
+            body,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        if (errorMessage.includes("407") || errorMessage.includes("authentication")) {
+            throw {
+                message: "Proxy authentication failed",
+                code: "PROXY_AUTH_FAILED",
+                originalError: error,
+            };
+        }
+        if (errorMessage.includes("ECONNREFUSED") || errorMessage.includes("ENOTFOUND")) {
+            throw {
+                message: "Proxy connection refused",
+                code: "PROXY_CONNECTION_REFUSED",
+                originalError: error,
+            };
+        }
+        throw error;
+    }
+}
+// check if url is accessible
+async function isUrlAccessible(url) {
+    try {
+        const response = await fetch(url, { method: "HEAD" });
+        return response.ok;
+    }
+    catch {
+        return false;
+    }
+}
+// clean up text by removing excessive whitespace and making it more readable
+function cleanText(text) {
+    if (!text)
+        return "";
+    return text
+        .replace(/[\n\s\r]+/g, " ")
+        .replace(/([.!?])\s+/g, "$1\n\n")
+        .trim();
+}
+// Helper function to get cache key
+function getCacheKey(query, options) {
+    return `${query}-${JSON.stringify(options)}`;
+}

package/dist/modules/common.test.d.ts

@@ -0,0 +1 @@
+export {};