llm-search-tools 1.1.0

package/docs/README.md

@@ -0,0 +1,275 @@
+# llm-search-tools Documentation
+
+This documentation covers usage for both TypeScript and Node.js environments.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [TypeScript Usage](#typescript-usage)
+- [Node.js Usage](#nodejs-usage)
+- [Module Documentation](#module-documentation)
+- [Scraper Module - Bot Detection & Proxies](#scraper-module---bot-detection--proxies)
+
+## Installation
+
+```bash
+npm install llm-search-tools
+```
+
+## TypeScript Usage
+
+### Basic Search
+
+```typescript
+import { search, SearchResult } from "llm-search-tools";
+
+async function searchExample() {
+    try {
+        const results: SearchResult[] = await search("typescript tutorial");
+        console.log(results);
+    } catch (error) {
+        console.error("Search failed:", error);
+    }
+}
+```
+
+### Wikipedia Search and Content
+
+```typescript
+import { wikiSearch, wikiGetContent, WikipediaResult } from "llm-search-tools";
+
+async function wikiExample() {
+    try {
+        const results: WikipediaResult[] = await wikiSearch("Node.js");
+        const content = await wikiGetContent(results[0].title);
+        console.log(content);
+    } catch (error) {
+        console.error("Wiki search failed:", error);
+    }
+}
+```
+
+### HackerNews Integration
+
+```typescript
+import {
+    getTopStories,
+    getNewStories,
+    getStoryById,
+    HackerNewsResult,
+} from "llm-search-tools";
+
+async function hnExample() {
+    try {
+        const topStories: HackerNewsResult[] = await getTopStories(5);
+        const newStories: HackerNewsResult[] = await getNewStories(5);
+        const story = await getStoryById(topStories[0].id);
+        console.log({ topStories, newStories, story });
+    } catch (error) {
+        console.error("HN fetch failed:", error);
+    }
+}
+```
+
+### Webpage Content Extraction
+
+```typescript
+import { getWebpageContent, WebpageContent } from "llm-search-tools";
+
+async function webpageExample() {
+    try {
+        const content: WebpageContent = await getWebpageContent(
+            "https://example.com",
+        );
+        console.log({
+            title: content.title,
+            text: content.textContent,
+            excerpt: content.excerpt,
+        });
+    } catch (error) {
+        console.error("Content extraction failed:", error);
+    }
+}
+```
+
+## Node.js Usage
+
+### Basic Search
+
+```javascript
+const { search } = require("llm-search-tools");
+
+async function searchExample() {
+    try {
+        const results = await search("nodejs tutorial");
+        console.log(results);
+    } catch (error) {
+        console.error("Search failed:", error);
+    }
+}
+```
+
+### Wikipedia Search and Content
+
+```javascript
+const { wikiSearch, wikiGetContent } = require("llm-search-tools");
+
+async function wikiExample() {
+    try {
+        const results = await wikiSearch("Node.js");
+        const content = await wikiGetContent(results[0].title);
+        console.log(content);
+    } catch (error) {
+        console.error("Wiki search failed:", error);
+    }
+}
+```
+
+### HackerNews Integration
+
+```javascript
+const { getTopStories, getNewStories, getStoryById } = require("llm-search-tools");
+
+async function hnExample() {
+    try {
+        const topStories = await getTopStories(5);
+        const newStories = await getNewStories(5);
+        const story = await getStoryById(topStories[0].id);
+        console.log({ topStories, newStories, story });
+    } catch (error) {
+        console.error("HN fetch failed:", error);
+    }
+}
+```
+
+### Webpage Content Extraction
+
+```javascript
+const { getWebpageContent } = require("llm-search-tools");
+
+async function webpageExample() {
+    try {
+        const content = await getWebpageContent("https://example.com");
+        console.log({
+            title: content.title,
+            text: content.textContent,
+            excerpt: content.excerpt,
+        });
+    } catch (error) {
+        console.error("Content extraction failed:", error);
+    }
+}
+```
+
+## Module Documentation
+
+For detailed documentation of each module, see:
+
+- [Scraper Module](./scraper.md) - Bot detection, proxy support, and advanced scraping
+- [Wikipedia Module](./wikipedia.md)
+- [HackerNews Module](./hackernews.md)
+- [Webpage Module](./webpage.md)
+
+## Scraper Module - Bot Detection & Proxies
+
+The scraper module provides intelligent web scraping with automatic bot detection and proxy support.
+
+### Quick Examples
+
+#### Basic Search with Bot Detection
+
+```typescript
+import { search } from "llm-search-tools";
+
+// Automatically handles bot protection
+const results = await search("typescript tutorial");
+```
+
+#### Using a Proxy
+
+```typescript
+// Proxy configuration
+const results = await search("nodejs tutorial", {
+    proxy: "http://proxy.example.com:8080",
+});
+
+// Or with authentication
+const results = await search("python tutorial", {
+    proxy: "http://user:pass@proxy.example.com:8080",
+});
+```
+
+#### Force Puppeteer for JavaScript-heavy Sites
+
+```typescript
+const results = await search("react tutorial", {
+    forcePuppeteer: true,
+    limit: 10,
+});
+```
+
+#### Advanced Configuration
+
+```typescript
+import { ProxyConfig, ScraperOptions } from "llm-search-tools";
+
+const proxyConfig: ProxyConfig = {
+    type: "socks5",
+    host: "proxy.example.com",
+    port: 1080,
+    auth: {
+        username: "user",
+        password: "pass",
+    },
+};
+
+const options: ScraperOptions = {
+    limit: 20,
+    proxy: proxyConfig,
+    antiBot: {
+        enabled: true,
+        maxRetries: 5,
+        retryDelay: 3000,
+    },
+};
+
+const results = await search("web scraping", options);
+```
+
+### Key Features
+
+- **Automatic Bot Detection**: Detects Cloudflare, PerimeterX, Akamai, DataDome, and more
+- **Puppeteer Fallback**: Seamlessly switches to browser automation when needed
+- **Full Proxy Support**: HTTP, HTTPS, SOCKS4, SOCKS5 with authentication
+- **Rate Limiting**: Built-in protection against IP bans
+- **Backward Compatible**: Works with existing search API code
+
+For complete documentation, see the [Scraper Module Documentation](./scraper.md).
+
+## Error Handling
+
+All functions throw a `SearchError` type with the following structure:
+
+```typescript
+interface SearchError {
+    message: string; // Human-readable error message
+    code: string; // Error code for programmatic handling
+    originalError?: unknown; // Original error object if available
+}
+```
+
+Example error handling:
+
+```typescript
+try {
+    const results = await search("typescript");
+} catch (error) {
+    if (error.code === "GOOGLE_SEARCH_ERROR") {
+        // Handle Google search error
+    } else if (error.code === "DDG_SEARCH_ERROR") {
+        // Handle DuckDuckGo error
+    }
+}
+```
+
+The same error handling works in JavaScript, just without the type annotations.

package/docs/autocomplete.md

@@ -0,0 +1,73 @@
+# Autocomplete Documentation
+
+The autocomplete module provides real-time search suggestions from multiple search engines. It's useful for building "search as you type" interfaces or generating keyword ideas.
+
+## Supported Providers
+
+- **Google**
+- **DuckDuckGo** (default)
+- **Yahoo**
+- **Brave**
+- **Yandex**
+- **Ecosia**
+- **Startpage**
+- **Qwant**
+- **Swisscows**
+
+## Usage
+
+```typescript
+import { getSuggestions } from "llm-search-tools";
+
+// Default (DuckDuckGo)
+const results = await getSuggestions("typescript");
+console.log(results.suggestions);
+// Output: ['typescript tutorial', 'typescript types', ...]
+
+// Specify provider
+const googleResults = await getSuggestions("react", "google");
+
+// With options
+const braveResults = await getSuggestions("linux", "brave", {
+    limit: 5,
+    proxy: "http://proxy.example.com:8080",
+});
+```
+
+## API Reference
+
+### `getSuggestions(query, provider?, options?)`
+
+- **query**: `string` - The partial search term
+- **provider**: `string` - One of the supported providers (default: `'duckduckgo'`)
+- **options**: `AutocompleteOptions`
+
+### Options
+
+| Option    | Type                    | Description                                                                       |
+| --------- | ----------------------- | --------------------------------------------------------------------------------- |
+| `limit`   | `number`                | Maximum number of suggestions to return (default varies by provider, usually ~10) |
+| `proxy`   | `string \| ProxyConfig` | Proxy configuration                                                               |
+| `timeout` | `number`                | Request timeout in milliseconds                                                   |
+
+## Return Value
+
+Returns a `Promise<AutocompleteResult>`:
+
+```typescript
+interface AutocompleteResult {
+    query: string; // The original query
+    suggestions: string[]; // Array of suggestion strings
+    source: string; // The provider used (e.g., 'google')
+}
+```
+
+## Error Handling
+
+```typescript
+try {
+    const results = await getSuggestions("query", "google");
+} catch (error) {
+    console.error("Autocomplete failed:", error.message);
+}
+```

package/docs/crawling.md

@@ -0,0 +1,88 @@
+# Crawling Module Documentation
+
+The crawling module allows you to crawl websites to extract content from multiple pages, following links up to a specified depth. It supports both lightweight HTML parsing (Cheerio) and full browser automation (Puppeteer).
+
+## Features
+
+- **Multi-page Crawling**: Follows links to discover new pages
+- **Configurable Depth**: Control how deep the crawler goes
+- **Domain Confinement**: Option to stay within the original domain
+- **Dual Modes**:
+    - `cheerio` (default): Fast, lightweight, static HTML only
+    - `puppeteer`: Full browser automation for dynamic JS-heavy sites
+- **Content Normalization**: Automatically converts pages to clean text and Markdown
+- **Proxy Support**: Route requests through proxies
+- **Stealth Mode**: Built-in evasion for bot protection when using Puppeteer
+
+## Usage
+
+```typescript
+import { crawl } from "llm-search-tools";
+
+// Basic crawl (depth 2, max 10 pages)
+const results = await crawl("https://example.com");
+
+results.forEach((page) => {
+    console.log(`URL: ${page.url}`);
+    console.log(`Title: ${page.title}`);
+    console.log(`Text: ${page.textContent.substring(0, 100)}...`);
+});
+```
+
+### Advanced Usage
+
+```typescript
+const results = await crawl("https://example.com", {
+    maxDepth: 3,
+    maxPages: 50,
+    stayOnDomain: true,
+    crawlType: "puppeteer", // Use for dynamic sites
+    proxy: "http://user:pass@proxy.com:8080",
+});
+```
+
+## Options
+
+The `crawl` function accepts a `CrawlOptions` object:
+
+| Option | Type | Default | Description |
+| hum | --- | --- | --- |
+| `maxDepth` | `number` | `2` | How many links deep to follow (0 = only start URL) |
+| `maxPages` | `number` | `10` | Maximum number of pages to crawl |
+| `stayOnDomain` | `boolean` | `true` | If true, only follows links on the same domain |
+| `crawlType` | `'cheerio' \| 'puppeteer'` | `'cheerio'` | Use `cheerio` for speed, `puppeteer` for dynamic content |
+| `forcePuppeteer` | `boolean` | `false` | Alias for setting `crawlType: 'puppeteer'` |
+| `proxy` | `string \| ProxyConfig` | `undefined` | Proxy configuration |
+
+## Output Structure
+
+The crawler returns a `Promise<CrawlResult>`, which is an array of `CrawledPage` objects:
+
+```typescript
+interface CrawledPage {
+    url: string;
+    title: string;
+    content: string; // HTML content
+    textContent: string; // Cleaned text
+    markdown?: string; // Markdown version
+    excerpt?: string;
+    siteName?: string;
+    favicon?: string;
+    imageUrls?: string[];
+    depth: number; // Depth at which this page was found
+}
+```
+
+## Error Handling
+
+The crawler is robust and will log errors for individual pages while continuing to crawl others. If the entire crawl fails to start, it throws a `SearchError`.
+
+```typescript
+try {
+    const results = await crawl("https://invalid-url.com");
+} catch (error) {
+    if (error.code === "CRAWL_ERROR") {
+        console.error("Crawling failed:", error.message);
+    }
+}
+```

package/docs/events.md

@@ -0,0 +1,58 @@
+# Events Search Documentation
+
+The events module enables searching for real-world events (concerts, festivals, meetups, etc.) using Google Events data. It uses Puppeteer with stealth techniques to extract structured event information.
+
+## Usage
+
+```typescript
+import { searchEvents } from "llm-search-tools";
+
+const results = await searchEvents("concerts in Tokyo");
+
+results.events.forEach((event) => {
+    console.log(`${event.date}: ${event.title} @ ${event.location}`);
+});
+```
+
+## API Reference
+
+### `searchEvents(query, options?)`
+
+- **query**: `string` - Search query (e.g., "tech conferences London", "music festivals 2024")
+- **options**: `EventSearchOptions`
+
+### Options
+
+| Option    | Type                    | Description                                      |
+| --------- | ----------------------- | ------------------------------------------------ |
+| `limit`   | `number`                | Maximum number of events to return (default: 10) |
+| `date`    | `string`                | _Experimental_ date filter                       |
+| `proxy`   | `string \| ProxyConfig` | Proxy configuration                              |
+| `timeout` | `number`                | Navigation timeout in ms (default: 30000)        |
+
+## Output Structure
+
+Returns a `Promise<EventResult>`:
+
+```typescript
+interface EventResult {
+    events: Event[];
+    url: string; // The Google Events URL scraped
+    source: string; // 'google-events'
+}
+
+interface Event {
+    title: string;
+    date: string;
+    location: string;
+    link?: string;
+    description?: string;
+    image?: string;
+}
+```
+
+## Notes
+
+- This module relies on DOM scraping of Google's "Events" UI (`ibp=htl;events`).
+- The structure of the Google Events page may change, which can affect extraction reliability.
+- Proxy usage is recommended for frequent scraping to avoid CAPTCHAs.

package/docs/examples.md

@@ -0,0 +1,158 @@
+# Examples 📚
+
+Here are some examples of how to use llm-search-tools in different scenarios.
+
+## Basic Search Example
+
+```typescript
+import { search, wikiSearch, getWebpageContent } from 'llm-search-tools';
+
+async function searchAndGetContent() {
+  try {
+    // Search across engines
+    const results = await search('typescript tutorial');
+    console.log('Search results:', results);
+
+    // Get webpage content from first result
+    if (results.length > 0) {
+      const content = await getWebpageContent(results[0].url);
+      console.log('First result content:', content.textContent);
+    }
+  } catch (err) {
+    console.error('Search failed:', err);
+  }
+}
+```
+
+## Wikipedia Research Example
+
+```typescript
+import { wikiSearch, wikiGetContent, wikiGetSummary } from 'llm-search-tools';
+
+async function researchTopic() {
+  try {
+    // Search Wikipedia
+    const results = await wikiSearch('artificial intelligence');
+    
+    // Get full article for first result
+    if (results.length > 0) {
+      const summary = await wikiGetSummary(results[0].title);
+      console.log('Summary:', summary.extract);
+      
+      const content = await wikiGetContent(results[0].title);
+      console.log('Full content:', content);
+    }
+  } catch (err) {
+    console.error('Wiki search failed:', err);
+  }
+}
+```
+
+## HackerNews Feed Example
+
+```typescript
+import { getTopStories, getBestStories, getStoryById } from 'llm-search-tools';
+
+async function getHNFeed() {
+  try {
+    // Get mix of top and best stories
+    const topStories = await getTopStories(5);
+    const bestStories = await getBestStories(5);
+    
+    console.log('Top stories:', topStories);
+    console.log('Best stories:', bestStories);
+    
+    // Get full details of first story
+    if (topStories.length > 0) {
+      const firstStory = await getStoryById(topStories[0].id);
+      console.log('Full story details:', firstStory);
+    }
+  } catch (err) {
+    console.error('HN fetch failed:', err);
+  }
+}
+```
+
+## Webpage Content Extraction Example
+
+```typescript
+import { getWebpageContent, getWebpageText, isUrlAccessible } from 'llm-search-tools';
+
+async function extractContent(url: string) {
+  try {
+    // Check if URL is accessible
+    if (await isUrlAccessible(url)) {
+      // Get full content with HTML
+      const content = await getWebpageContent(url);
+      console.log('Title:', content.title);
+      console.log('Excerpt:', content.excerpt);
+      console.log('Content:', content.content);
+      
+      // Get just the text
+      const text = await getWebpageText(url);
+      console.log('Plain text:', text);
+    }
+  } catch (err) {
+    console.error('Content extraction failed:', err);
+  }
+}
+```
+
+## Multi-Source Research Example
+
+```typescript
+import { search, wikiSearch, getWebpageContent, getTopStories } from 'llm-search-tools';
+
+async function researchTopic(query: string) {
+  try {
+    // Search multiple sources in parallel
+    const [
+      searchResults,
+      wikiResults,
+      hnStories
+    ] = await Promise.all([
+      search(query),
+      wikiSearch(query),
+      getTopStories(5)
+    ]);
+    
+    console.log('Web search results:', searchResults);
+    console.log('Wikipedia results:', wikiResults);
+    console.log('Related HN stories:', hnStories);
+    
+    // Get content from first search result
+    if (searchResults.length > 0) {
+      const content = await getWebpageContent(searchResults[0].url);
+      console.log('Main article content:', content.textContent);
+    }
+  } catch (err) {
+    console.error('Research failed:', err);
+  }
+}
+```
+
+## Error Handling Example
+
+```typescript
+import { search, SearchError } from 'llm-search-tools';
+
+async function robustSearch(query: string) {
+  try {
+    const results = await search(query);
+    return results;
+  } catch (err) {
+    if (err.code === 'GOOGLE_SEARCH_ERROR') {
+      console.log('Google search failed, using fallback...');
+      // Handle error or try alternative
+    } else if (err.code === 'DDG_SEARCH_ERROR') {
+      console.log('DuckDuckGo search failed...');
+      // Handle error
+    } else {
+      console.error('Unknown error:', err);
+    }
+    return [];
+  }
+}
+```
+
+These examples demonstrate some common use cases, but there are many more possibilities! Check out the module-specific documentation for more details on available functions and options.

package/docs/finance.md

@@ -0,0 +1,60 @@
+# Finance Module 📈
+
+The finance module provides real-time stock quotes and financial market data using Yahoo Finance.
+
+## Functions
+
+### getQuote(symbol: string)
+
+Main finance function that retrieves stock quotes. Currently backed by Yahoo Finance.
+
+```typescript
+import { getQuote } from "llm-search-tools";
+
+const quote = await getQuote("AAPL");
+console.log(`${quote.symbol}: $${quote.regularMarketPrice}`);
+```
+
+### getStockQuote(symbol: string)
+
+Direct access to the Yahoo Finance scraper.
+
+```typescript
+import { getStockQuote } from "llm-search-tools";
+
+const quote = await getStockQuote("GOOGL");
+```
+
+## Result Format
+
+The module returns `FinanceResult` objects:
+
+```typescript
+interface FinanceResult {
+  symbol: string;
+  shortName?: string;
+  longName?: string;
+  regularMarketPrice?: number;
+  regularMarketChange?: number;
+  regularMarketChangePercent?: number;
+  regularMarketTime?: Date;
+  currency?: string;
+  exchange?: string;
+  marketState?: string;
+  source: "yahoo-finance";
+}
+```
+
+## Error Handling
+
+If the symbol is not found or the API fails, the function throws a `SearchError` with code `FINANCE_QUOTE_ERROR`.
+
+```typescript
+try {
+  const quote = await getQuote("INVALID_SYMBOL");
+} catch (err) {
+  if (err.code === "FINANCE_QUOTE_ERROR") {
+    console.error("Failed to fetch quote:", err.message);
+  }
+}
+```