llm-search-tools 1.1.0

package/dist/scripts/interactive-search.js

@@ -0,0 +1,98 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const readline = __importStar(require("readline"));
+const google_1 = require("../modules/scrapers/google");
+const duckduckgo_1 = require("../modules/scrapers/duckduckgo");
+const searxng_1 = require("../modules/scrapers/searxng");
+const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout
+});
+const searchEngine = async (name, searchFn, query) => {
+    try {
+        console.log(`\n🔍 Searching ${name}...`);
+        const results = await searchFn(query, { limit: 3 });
+        console.log(`\n✅ ${name} Results:`);
+        if (results.length === 0) {
+            console.log('  No results found.');
+        }
+        else {
+            results.slice(0, 3).forEach((result, index) => {
+                console.log(`  ${index + 1}. ${result.title}`);
+                console.log(`     ${result.url}`);
+                if (result.snippet)
+                    console.log(`     "${result.snippet.substring(0, 100)}..."`);
+            });
+        }
+    }
+    catch (error) {
+        let errorMessage;
+        if (error instanceof Error) {
+            errorMessage = error.message;
+        }
+        else if (typeof error === 'object' && error !== null) {
+            try {
+                errorMessage = JSON.stringify(error, null, 2);
+            }
+            catch {
+                errorMessage = String(error);
+            }
+        }
+        else {
+            errorMessage = String(error);
+        }
+        console.log(`\n❌ ${name} Failed: ${errorMessage}`);
+    }
+};
+console.log("Welcome to the LLM-Search-Tools Interactive Tester!");
+console.log("This tool will search DuckDuckGo, Google, and SearxNG for your query.");
+rl.question('\n👉 Enter what to search: ', async (query) => {
+    if (!query.trim()) {
+        console.log("Empty query, exiting.");
+        rl.close();
+        return;
+    }
+    console.log(`\n🚀 Starting search for: "${query}"...`);
+    // Run sequentially to keep output clean, or parallel?
+    // Parallel might mix output. Let's do parallel but capture output, or just sequential for readability.
+    // User asked to "search all search engines", sequential is fine and cleaner for reading.
+    await searchEngine('DuckDuckGo', duckduckgo_1.searchDuckDuckGo, query);
+    await searchEngine('Google', google_1.searchGoogle, query);
+    await searchEngine('SearxNG', searxng_1.searchSearxNG, query);
+    console.log("\n✨ All searches completed!");
+    rl.close();
+    process.exit(0);
+});

package/dist/test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test.js

@@ -0,0 +1,179 @@
+"use strict";
+// test.ts - test all the things cuz we're responsible devs... kinda
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = require("fs");
+const path_1 = require("path");
+const index_1 = require("./index");
+// get random item from array
+function getRandomItem(arr) {
+    return arr[Math.floor(Math.random() * arr.length)];
+}
+// delay helper
+const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+// test parsing all files in test/files directory
+async function testParser() {
+    console.log('\n📝 PARSER MODULE TEST');
+    console.log('-------------------');
+    const testFilesDir = (0, path_1.join)(process.cwd(), 'test', 'files');
+    const files = (0, fs_1.readdirSync)(testFilesDir);
+    for (const file of files) {
+        const filePath = (0, path_1.join)(testFilesDir, file);
+        console.log(`\nTesting file: ${file}`);
+        try {
+            const result = await (0, index_1.parse)(filePath, {
+                csv: { columns: true },
+                language: 'eng',
+                xml: {
+                    ignoreAttributes: false,
+                    parseAttributeValue: true
+                }
+            });
+            console.log('File type:', result.type);
+            console.log('Metadata:', JSON.stringify(result.metadata, null, 2));
+            if (result.data) {
+                console.log('Data sample:', JSON.stringify(result.data?.slice?.(0, 2) || result.data, null, 2));
+            }
+            // Show preview of text content
+            const textPreview = result.text.slice(0, 200) + (result.text.length > 200 ? '...' : '');
+            console.log('Text preview:', textPreview);
+            console.log('✅ Successfully parsed');
+        }
+        catch (err) {
+            const error = err;
+            console.warn(`⚠️ Failed to parse ${file}:`, error.message);
+            if (error.code) {
+                console.warn('Error code:', error.code);
+            }
+        }
+    }
+}
+async function runTests() {
+    console.log('🔍 starting tests...\n');
+    try {
+        // test scraper module
+        console.log('📖 SCRAPER MODULE TEST');
+        console.log('---------------------');
+        console.log('\n1. Basic Search Results:');
+        const searchResults = await (0, index_1.search)('typescript programming');
+        console.log(JSON.stringify(searchResults, null, 2));
+        await delay(3000);
+        console.log('\n2. Google Search Results:');
+        const googleResults = await (0, index_1.searchGoogle)('react hooks');
+        console.log(JSON.stringify(googleResults, null, 2));
+        await delay(3000);
+        console.log('\n3. DuckDuckGo Search Results:');
+        const ddgResults = await (0, index_1.searchDuckDuckGo)('node.js async');
+        console.log(JSON.stringify(ddgResults, null, 2));
+        await delay(3000);
+        console.log('\n4. Search with Puppeteer Forced:');
+        const puppeteerResults = await (0, index_1.search)('web scraping', {
+            forcePuppeteer: true,
+            limit: 5
+        });
+        console.log(JSON.stringify(puppeteerResults, null, 2));
+        await delay(5000);
+        console.log('\n5. Search with Proxy (if available):');
+        // Example proxy config - replace with actual proxy for testing
+        const proxyConfig = {
+            type: 'http',
+            host: 'proxy.example.com',
+            port: 8080,
+            auth: {
+                username: 'user',
+                password: 'pass'
+            }
+        };
+        // Only test proxy if you have one available
+        // const proxyResults = await search('test query', {
+        //   proxy: proxyConfig,
+        //   limit: 3
+        // });
+        // console.log(JSON.stringify(proxyResults, null, 2));
+        console.log('Proxy test skipped - configure proxy to test');
+        await delay(3000);
+        // test wikipedia module
+        console.log('\n\n📚 WIKIPEDIA MODULE TEST');
+        console.log('----------------------');
+        console.log('\n1. Wikipedia Search Results:');
+        const wikiResults = await (0, index_1.wikiSearch)('Node.js');
+        console.log(JSON.stringify(wikiResults, null, 2));
+        if (wikiResults.length > 0) {
+            await delay(1000);
+            console.log('\n2. Wikipedia Article Content:');
+            const content = await (0, index_1.wikiGetContent)(wikiResults[0].title);
+            console.log(content.substring(0, 500) + '...');
+            await delay(1000);
+            console.log('\n3. Wikipedia Article Summary:');
+            const summary = await (0, index_1.wikiGetSummary)(wikiResults[0].title);
+            console.log(JSON.stringify(summary, null, 2));
+        }
+        // Add delay before next module
+        await delay(2000);
+        // test hackernews module
+        console.log('\n\n💻 HACKER NEWS MODULE TEST');
+        console.log('------------------------');
+        console.log('\n1. Top Stories:');
+        const topStories = await (0, index_1.getTopStories)(5);
+        console.log(JSON.stringify(topStories, null, 2));
+        await delay(500);
+        console.log('\n2. New Stories:');
+        const newStories = await (0, index_1.getNewStories)(5);
+        console.log(JSON.stringify(newStories, null, 2));
+        await delay(500);
+        console.log('\n3. Best Stories:');
+        const bestStories = await (0, index_1.getBestStories)(5);
+        console.log(JSON.stringify(bestStories, null, 2));
+        await delay(500);
+        console.log('\n4. Ask HN Stories:');
+        const askStories = await (0, index_1.getAskStories)(5);
+        console.log(JSON.stringify(askStories, null, 2));
+        await delay(500);
+        console.log('\n5. Show HN Stories:');
+        const showStories = await (0, index_1.getShowStories)(5);
+        console.log(JSON.stringify(showStories, null, 2));
+        await delay(500);
+        console.log('\n6. Job Stories:');
+        const jobStories = await (0, index_1.getJobStories)(5);
+        console.log(JSON.stringify(jobStories, null, 2));
+        if (topStories.length > 0) {
+            await delay(500);
+            console.log('\n7. Single Story Details:');
+            const storyId = parseInt(topStories[0].url.split('id=')[1]);
+            const story = await (0, index_1.getStoryById)(storyId);
+            console.log(JSON.stringify(story, null, 2));
+        }
+        // Add delay before next module
+        await delay(2000);
+        // test webpage module
+        console.log('\n\n📄 WEBPAGE MODULE TEST');
+        console.log('--------------------');
+        const testUrl = 'https://example.com';
+        console.log(`\n1. Testing URL: ${testUrl}`);
+        const isAccessible = await (0, index_1.isUrlAccessible)(testUrl);
+        console.log('\nURL Accessibility:', isAccessible);
+        if (isAccessible) {
+            console.log('\n2. Webpage Content:');
+            const content = await (0, index_1.getWebpageContent)(testUrl);
+            console.log(JSON.stringify(content, null, 2));
+            await delay(1000);
+            console.log('\n3. Webpage Text:');
+            const text = await (0, index_1.getWebpageText)(testUrl);
+            console.log(text);
+        }
+        // Add delay before parser tests
+        await delay(2000);
+        // test parser module with all test files
+        await testParser();
+        console.log('\n\n🎉 ALL TESTS COMPLETED SUCCESSFULLY!\n');
+    }
+    catch (err) {
+        const error = err;
+        console.error('\n❌ TEST FAILED:', {
+            message: error.message,
+            code: error.code,
+            details: error.originalError
+        });
+        process.exit(1);
+    }
+}
+runTests();

package/dist/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../src/test.ts"],"names":[],"mappings":";AAAA,oEAAoE;;AAEpE,mCAiBiB;AAEjB,6BAA6B;AAC7B,SAAS,aAAa,CAAI,GAAQ;IAChC,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,QAAQ;IACrB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEtC,IAAI,CAAC;QACH,qBAAqB;QACrB,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,MAAM,IAAA,cAAM,EAAC,YAAY,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEpD,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,MAAM,aAAa,GAAG,MAAM,IAAA,oBAAY,EAAC,YAAY,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEpD,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAA,wBAAgB,EAAC,YAAY,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjD,wBAAwB;QACxB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAEtC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,MAAM,IAAA,kBAAU,EAAC,SAAS,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAc,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;YAE/C,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,IAAA,sBAAc,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,yBAAyB;QACzB,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjD,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,IAAA,sBAAc,EAAC,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,MAAM,IAAA,sBAAc,EAAC,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAA,qBAAa,EAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAY,EAAC,OAAO,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,gDAAgD;QAChD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAEpC,8CAA8C;QAC9C,MAAM,YAAY,GAAG,aAAa,CAAC,aAAa,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,yCAAyC,YAAY,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzE,MAAM,YAAY,GAAG,MAAM,IAAA,uBAAe,EAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,YAAY,CAAC,CAAC;QAElD,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,IAAA,yBAAiB,EAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAE9C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,IAAA,sBAAc,EAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;YAChD,MAAM,WAAW,GAAG,qBAAqB,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,IAAA,yBAAiB,EAAC,WAAW,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAE9C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,IAAA,sBAAc,EAAC,WAAW,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAE5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,QAAQ,EAAE,CAAC"}

package/dist/testBraveSearch.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/testBraveSearch.js

@@ -0,0 +1,34 @@
+"use strict";
+// test brave search separately
+Object.defineProperty(exports, "__esModule", { value: true });
+const search_1 = require("./modules/search");
+async function testBraveSearch() {
+    try {
+        console.log('🔍 Testing Brave Search...\n');
+        const query = 'typescript programming';
+        console.log(`Searching for: "${query}"`);
+        const results = await (0, search_1.search)(query, {
+            provider: 'brave',
+            limit: 5,
+            safeSearch: true
+        });
+        if (results.length === 0) {
+            console.log('No results found');
+        }
+        else {
+            console.log(`\nFound ${results.length} results:`);
+            results.forEach((result, index) => {
+                console.log(`\n${index + 1}. ${result.title}`);
+                console.log(`   URL: ${result.url}`);
+                if (result.snippet) {
+                    console.log(`   Snippet: ${result.snippet}`);
+                }
+            });
+        }
+    }
+    catch (err) {
+        console.error('\n❌ Test failed:', err);
+        process.exit(1);
+    }
+}
+testBraveSearch().catch(console.error);

package/dist/testDuckDuckGo.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/testDuckDuckGo.js

@@ -0,0 +1,52 @@
+"use strict";
+// Test DuckDuckGo search in isolation
+Object.defineProperty(exports, "__esModule", { value: true });
+const search_1 = require("./modules/search");
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function testDuckDuckGo() {
+    try {
+        console.log('🔍 Testing DuckDuckGo Search...\n');
+        // Test regular search
+        const query = 'typescript programming';
+        console.log(`1. Regular Search for: "${query}"`);
+        const results = await (0, search_1.search)(query, {
+            provider: 'duckduckgo',
+            limit: 5
+        });
+        if (results.length === 0) {
+            console.log('No results found');
+        }
+        else {
+            console.log(`\nFound ${results.length} results:`);
+            results.forEach((result, index) => {
+                console.log(`\n${index + 1}. ${result.title}`);
+                console.log(`   URL: ${result.url}`);
+                if (result.snippet) {
+                    console.log(`   Snippet: ${result.snippet}`);
+                }
+            });
+        }
+        // Wait before safe search test
+        await delay(5000);
+        console.log('\n2. SafeSearch Test');
+        const safeResults = await (0, search_1.search)('adult content', {
+            provider: 'duckduckgo',
+            limit: 3,
+            safeSearch: true
+        });
+        console.log('\nSafe Search Results:');
+        safeResults.forEach((result, index) => {
+            console.log(`\n${index + 1}. ${result.title}`);
+        });
+        console.log('\n✅ DuckDuckGo tests completed successfully!');
+    }
+    catch (err) {
+        const error = err;
+        console.error('\n❌ Test failed:', error.message);
+        if (error.code === 'DDG_SEARCH_ERROR') {
+            console.log('\nTip: DuckDuckGo has strict rate limits. Try waiting a few minutes before retrying.');
+        }
+        process.exit(1);
+    }
+}
+testDuckDuckGo().catch(console.error);

package/dist/testEcosia.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/testEcosia.js

@@ -0,0 +1,57 @@
+"use strict";
+// Test Ecosia search in isolation
+Object.defineProperty(exports, "__esModule", { value: true });
+const search_1 = require("./modules/search");
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function testEcosiaSearch() {
+    try {
+        console.log('🔍 Testing Ecosia Search...\n');
+        // Test regular search
+        const query = 'typescript programming';
+        console.log(`1. Regular Search for: "${query}"`);
+        const results = await (0, search_1.search)(query, {
+            provider: 'ecosia',
+            limit: 5
+        });
+        if (results.length === 0) {
+            console.log('No results found');
+        }
+        else {
+            console.log(`\nFound ${results.length} results:`);
+            results.forEach((result, index) => {
+                console.log(`\n${index + 1}. ${result.title}`);
+                console.log(`   URL: ${result.url}`);
+                if (result.snippet) {
+                    console.log(`   Snippet: ${result.snippet}`);
+                }
+            });
+        }
+        // Test different search term
+        await delay(2000);
+        console.log('\n2. Different Search Term Test');
+        const secondResults = await (0, search_1.search)('web development', {
+            provider: 'ecosia',
+            limit: 3
+        });
+        console.log('\nSearch Results:');
+        secondResults.forEach((result, index) => {
+            console.log(`\n${index + 1}. ${result.title}`);
+            console.log(`   URL: ${result.url}`);
+        });
+        console.log('\n✅ Ecosia tests completed successfully!');
+    }
+    catch (err) {
+        const error = err;
+        console.error('\n❌ Test failed:', error.message);
+        if (error.code === 'ECOSIA_SEARCH_ERROR') {
+            console.log('\nError details:', error.originalError?.message || 'Unknown error');
+            console.log('\nTips:');
+            console.log('1. Check if Ecosia is accessible in your region');
+            console.log('2. Verify your internet connection');
+            console.log('3. Try adjusting the delay between requests');
+        }
+        console.log('\nFull error:', JSON.stringify(error, null, 2));
+        process.exit(1);
+    }
+}
+testEcosiaSearch().catch(console.error);

package/dist/testSearchModule.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/testSearchModule.js

@@ -0,0 +1,95 @@
+"use strict";
+// Test search module separately to handle different rate limits
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function testSearch() {
+    try {
+        console.log("📖 SEARCH MODULE TESTING");
+        console.log("----------------------");
+        const testQuery = "typescript programming";
+        const testLimit = 3;
+        // Test Google
+        console.log("\n1. Google Search Results:");
+        const googleResults = await (0, index_1.search)(testQuery, {
+            provider: 'google',
+            limit: testLimit
+        });
+        console.log("Found", googleResults.length, "Google results");
+        console.log(JSON.stringify(googleResults, null, 2));
+        // Test Brave with delay
+        await delay(3000);
+        console.log("\nWaiting before Brave search...");
+        console.log("\n2. Brave Search Results:");
+        const braveResults = await (0, index_1.search)(testQuery, {
+            provider: 'brave',
+            limit: testLimit
+        });
+        console.log("Found", braveResults.length, "Brave results");
+        console.log(JSON.stringify(braveResults, null, 2));
+        // Test Ecosia with delay
+        await delay(3000);
+        console.log("\nWaiting before Ecosia search...");
+        console.log("\n3. Ecosia Search Results:");
+        const ecosiaResults = await (0, index_1.search)(testQuery, {
+            provider: 'ecosia',
+            limit: testLimit
+        });
+        console.log("Found", ecosiaResults.length, "Ecosia results");
+        console.log(JSON.stringify(ecosiaResults, null, 2));
+        // Test DuckDuckGo with longer delay
+        await delay(5000);
+        console.log("\nWaiting before DuckDuckGo search...");
+        console.log("\n4. DuckDuckGo Search Results:");
+        try {
+            const ddgResults = await (0, index_1.search)(testQuery, {
+                provider: 'duckduckgo',
+                limit: testLimit
+            });
+            console.log("Found", ddgResults.length, "DuckDuckGo results");
+            console.log(JSON.stringify(ddgResults, null, 2));
+            // Test SafeSearch features
+            await delay(5000);
+            console.log("\n5. SafeSearch Tests:");
+            // Test DuckDuckGo SafeSearch
+            console.log("\na) DuckDuckGo with SafeSearch:");
+            const ddgSafeResults = await (0, index_1.search)("adult content", {
+                provider: 'duckduckgo',
+                limit: 1,
+                safeSearch: true
+            });
+            console.log("DuckDuckGo SafeSearch result:", ddgSafeResults.length ? ddgSafeResults[0].title : "No results");
+            // Test Ecosia (always safe)
+            await delay(3000);
+            console.log("\nb) Ecosia Search (always safe):");
+            const ecosiaSafeResults = await (0, index_1.search)("adult content", {
+                provider: 'ecosia',
+                limit: 1
+            });
+            console.log("Ecosia filtered result:", ecosiaSafeResults.length ? ecosiaSafeResults[0].title : "No results");
+        }
+        catch (err) {
+            const error = err;
+            console.log("DuckDuckGo search failed (rate limiting):", error.message);
+            console.log("\nTry running search providers separately:");
+            console.log("- npm run test-ddg");
+            console.log("- npm run test-brave");
+            console.log("\nNote: Search providers have different rate limits:");
+            console.log("- DuckDuckGo: 5 seconds between requests");
+            console.log("- Ecosia: 2 seconds between requests");
+            console.log("- Brave/Google: 1 second between requests");
+        }
+        console.log("\n✅ Search module tests completed!");
+    }
+    catch (err) {
+        const error = err;
+        console.error("\n❌ TEST FAILED:", {
+            message: error.message,
+            code: error.code,
+            details: error.originalError,
+        });
+        process.exit(1);
+    }
+}
+// Run tests
+testSearch().catch(console.error);

package/dist/testwebpage.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/testwebpage.js

@@ -0,0 +1,81 @@
+"use strict";
+// ah yes, the test file... where we make sure this actually works n stuff
+Object.defineProperty(exports, "__esModule", { value: true });
+const webpage_1 = require("./modules/webpage");
+// just print stuff nicely ya know
+function printResult(title, content) {
+    console.log('\n' + '='.repeat(50));
+    console.log(`${title}:`);
+    console.log('='.repeat(50));
+    if (typeof content === 'string') {
+        console.log(content);
+    }
+    else {
+        console.log('Title:', content.title);
+        console.log('Excerpt:', content.excerpt);
+        console.log('Length:', content.length);
+        console.log('Site Name:', content.siteName || 'N/A');
+        console.log('Favicon:', content.faviconUrl || 'N/A');
+        if (content.assets && content.assets.length > 0) {
+            console.log('\nAssets found:');
+            content.assets.forEach(asset => {
+                console.log(`- ${asset.type}: ${asset.url} (${asset.alt || 'no alt text'})`);
+            });
+        }
+        console.log('\nFull Content:');
+        console.log(content.textContent);
+        if (content.markdown) {
+            console.log('\nFull Markdown:');
+            console.log(content.markdown);
+        }
+        if (content.content) {
+            console.log('\nFull HTML Content:');
+            console.log(content.content);
+        }
+        if (content.processedHtml) {
+            console.log('\nProcessed HTML:');
+            console.log(content.processedHtml);
+        }
+    }
+}
+// actually test the stuff
+async function runTests() {
+    try {
+        const url = process.argv[2];
+        if (!url) {
+            console.log('yo, gimme a url to test! like: npm run test-webpage https://example.com');
+            process.exit(1);
+        }
+        // lets try to get the content even if the url seems inaccessible
+        // some sites block HEAD requests but still work fine
+        console.log('checking if url is accessible...');
+        const isAccessible = await (0, webpage_1.isUrlAccessible)(url);
+        if (!isAccessible) {
+            console.log('heads up: url seems blocked, but well try anyway...');
+        }
+        // test getting content
+        console.log('\ntesting content extraction...');
+        try {
+            const content = await (0, webpage_1.getWebpageContent)(url);
+            printResult('Page Content', content);
+        }
+        catch (err) {
+            console.log('failed to get content:', err);
+        }
+        // test getting just the text
+        console.log('\ntesting text-only extraction...');
+        try {
+            const textContent = await (0, webpage_1.getWebpageText)(url);
+            printResult('Text-only Content', textContent);
+        }
+        catch (err) {
+            console.log('failed getting text:', err);
+        }
+    }
+    catch (err) {
+        console.error('something went really wrong:', err);
+        process.exit(1);
+    }
+}
+// lets do this!
+runTests().catch(console.error);