enterprise-ai-recursive-web-scraper 1.0.0

package/lib/index.d.cts

@@ -0,0 +1,1303 @@
+import { Browser } from 'puppeteer';
+import { GoogleGenerativeAI, HarmCategory, HarmBlockThreshold } from '@google/generative-ai';
+
+/**
+ * @fileoverview Enhanced web scraping and content filtering functions for detecting and filtering inappropriate content
+ * @file scraper.ts
+ * @module scraper
+ * @description This module provides functionality for web scraping with content filtering capabilities.
+ * It includes classes for managing browser operations, text processing, and content filtering using
+ * Trie data structures. The module is designed to detect and filter NSFW content, slurs, and other
+ * inappropriate content from web pages.
+ *
+ * Key features:
+ * - Web scraping using Puppeteer with stealth and ad-blocking capabilities
+ * - Content filtering using Trie data structures for efficient pattern matching
+ * - Text processing with duplicate detection and removal
+ * - NSFW domain detection and filtering
+ * - Configurable content replacement
+ *
+ * Classes:
+ * - ContentTrie: Trie data structure for efficient string matching
+ * - ContentFilterManager: Singleton manager for content filtering operations
+ * - TextProcessor: Text cleaning and duplicate detection utility
+ * - BrowserManager: Browser and page management for scraping
+ *
+ * @example
+ * ```typescript
+ * // Initialize the filtering system
+ * initializeFilterWords();
+ *
+ * // Scrape and filter content from a URL
+ * const result = await scrape('https://example.com');
+ * if ('error' in result) {
+ *   console.error(result.error);
+ * } else {
+ *   console.log(result.filteredTexts);
+ * }
+ *
+ * // Filter individual text
+ * const filtered = filterText('text to filter');
+ * ```
+ *
+ * @requires puppeteer-extra - Enhanced version of Puppeteer with plugin support
+ * @requires puppeteer-extra-plugin-adblocker - Plugin for blocking ads and trackers
+ * @requires puppeteer-extra-plugin-stealth - Plugin for avoiding bot detection
+ *
+ * @license MIT
+ * @author Original author and contributors
+ * @version 1.0.0
+ * @since 1.0.0
+ *
+ * @see {@link https://github.com/berstend/puppeteer-extra|puppeteer-extra} - Enhanced version of Puppeteer
+ * @see {@link https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth|puppeteer-extra-plugin-stealth} - Stealth plugin for avoiding detection
+ * @see {@link https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-adblocker|puppeteer-extra-plugin-adblocker} - Ad blocking plugin
+ *
+ * @todo Add support for custom filtering rules
+ * @todo Improve error handling and recovery
+ * @todo Add rate limiting and request throttling
+ * @todo Implement caching for frequently accessed content
+ * @todo Add support for proxy rotation
+ * @todo Improve duplicate detection algorithms
+ * @todo Add support for custom content processors
+ * @todo Implement better logging and monitoring
+ * @todo Add support for distributed scraping
+ * @todo Improve memory management for large-scale scraping
+ *
+ * @throws {Error} When filter initialization fails
+ * @throws {Error} When browser operations fail
+ * @throws {Error} When content processing fails
+ * @throws {Error} When network operations fail
+ *
+ * @property {ContentFilterManager} filterManager - Singleton instance for content filtering
+ * @property {BrowserManager} browserManager - Static class for browser operations
+ * @property {TextProcessor} textProcessor - Static class for text processing
+ *
+ * @borrows ContentFilterManager.filterText as filterText
+ * @borrows ContentFilterManager.getInstance as getFilterManager
+ * @borrows BrowserManager.launch as launchBrowser
+ * @borrows TextProcessor.processText as processText
+ *
+ * @exports scrape - Main scraping function
+ * @exports initializeFilterWords - Filter initialization function
+ * @exports filterText - Text filtering function
+ * @exports ContentFilterManager - Content filtering manager class
+ *
+ * @typedef {Object} ScrapingResult
+ * @property {boolean} [flaggedDomain] - Whether the domain is flagged as NSFW
+ * @property {boolean} [containsCensored] - Whether censored content was found
+ * @property {string[]} [filteredTexts] - Array of filtered text content
+ * @property {string} [error] - Error message if scraping failed
+ *
+ * @typedef {Object} CodeBlock
+ * @property {string} language - Programming language of the code block
+ * @property {string} code - The actual code content
+ * @property {boolean} lineNumbers - Whether line numbers should be displayed
+ *
+ * @typedef {Object} TrieNode
+ * @property {Object.<string, TrieNode>} children - Child nodes in the Trie
+ * @property {boolean} isEndOfWord - Whether this node represents end of word
+ *
+ * @typedef {Object} ContentExtractionResult
+ * @property {string[]} texts - Array of extracted text content
+ * @property {CodeBlock[]} codeBlocks - Array of extracted code blocks
+ *
+ * @typedef {Object} FilterOptions
+ * @property {string} [replacement="***"] - Replacement string for filtered content
+ * @property {boolean} [caseSensitive=false] - Whether filtering is case sensitive
+ * @property {number} [minLength=1] - Minimum length for content to be filtered
+ *
+ * @typedef {Object} BrowserOptions
+ * @property {boolean} [headless=true] - Whether to run browser in headless mode
+ * @property {string[]} [args] - Additional browser launch arguments
+ * @property {number} [timeout=30000] - Navigation timeout in milliseconds
+ *
+ * @typedef {Object} ProcessingOptions
+ * @property {number} [similarityThreshold=0.85] - Threshold for duplicate detection
+ * @property {number} [maxLength=50000] - Maximum content length for processing
+ * @property {boolean} [preserveFormatting=false] - Whether to preserve text formatting
+ */
+
+/**
+ * Singleton class managing content filtering operations.
+ * @class
+ * @description Provides centralized content filtering functionality using multiple
+ * filtering mechanisms including Tries and Sets. Implements the Singleton pattern
+ * to ensure consistent filtering across the application.
+ *
+ * Key features:
+ * - Singleton pattern ensures consistent filtering state
+ * - Multiple filtering mechanisms (Tries, Sets)
+ * - Configurable content replacement
+ * - Efficient text chunk processing
+ * - NSFW domain detection
+ *
+ * @example
+ * ```typescript
+ * const filterManager = ContentFilterManager.getInstance();
+ *
+ * // Check domain
+ * const isNSFW = filterManager.isNSFWDomain('example.com');
+ *
+ * // Filter text
+ * const filtered = filterManager.filterText('text to filter');
+ * ```
+ */
+declare class ContentFilterManager {
+    /**
+     * Singleton instance
+     * @private
+     * @static
+     * @type {ContentFilterManager}
+     */
+    private static instance;
+    /**
+     * Trie for storing and matching filtered words
+     * @private
+     * @type {ContentTrie}
+     */
+    private filterTrie;
+    /**
+     * Set of NSFW domains
+     * @private
+     * @type {Set<string>}
+     */
+    private nsfwDomains;
+    /**
+     * Trie for storing and matching NSFW terms
+     * @private
+     * @type {ContentTrie}
+     */
+    private nsfwNamesTrie;
+    /**
+     * Set of filtered dictionary words
+     * @private
+     * @type {Set<string>}
+     */
+    private filterDict;
+    /**
+     * Maximum content length for processing
+     * @private
+     * @readonly
+     * @type {number}
+     */
+    private readonly MAX_CONTENT_LENGTH;
+    /**
+     * Private constructor to prevent direct instantiation.
+     * @private
+     * @description Initializes all filtering data structures and loads initial data.
+     * This constructor is private to enforce the singleton pattern.
+     *
+     * @throws {Error} If filter initialization fails
+     */
+    private constructor();
+    /**
+     * Gets or creates the singleton instance of ContentFilterManager.
+     * @returns {ContentFilterManager} The singleton instance
+     * @description Ensures only one instance of ContentFilterManager exists.
+     * Creates the instance if it doesn't exist, otherwise returns the existing instance.
+     *
+     * @example
+     * ```typescript
+     * const filterManager = ContentFilterManager.getInstance();
+     * ```
+     */
+    static getInstance(): ContentFilterManager;
+    /**
+     * Loads filter data from configuration files.
+     * @private
+     * @returns {Promise<void>}
+     * @description Asynchronously loads filtering data from configuration files,
+     * including NSFW domains, NSFW terms, and slurs. Initializes all filtering
+     * data structures with the loaded data.
+     *
+     * @throws {Error} If filter initialization fails or data files cannot be loaded
+     */
+    private loadFilters;
+    /**
+     * Checks if a URL contains or belongs to an NSFW domain.
+     * @param {string} url - The URL to check
+     * @returns {boolean} True if the URL matches any NSFW domain patterns
+     * @description Performs case-sensitive matching against known NSFW domains.
+     * Checks if the URL contains any known NSFW domain patterns.
+     *
+     * @example
+     * ```typescript
+     * const filterManager = ContentFilterManager.getInstance();
+     * const isNSFW = filterManager.isNSFWDomain('example.com');
+     * ```
+     *
+     * @throws {TypeError} If url is not a string
+     */
+    isNSFWDomain(url: string): boolean;
+    /**
+     * Splits text into manageable chunks while preserving context.
+     * @private
+     * @param {string} text - Text to split
+     * @returns {string[]} Array of text chunks
+     * @description Splits long text into smaller chunks while trying to maintain
+     * sentence boundaries and context. This ensures efficient processing of large
+     * text content.
+     *
+     * @throws {TypeError} If text is not a string
+     */
+    private splitIntoChunks;
+    /**
+     * Filters text content using content filtering rules.
+     * @param {string} text - Text to filter
+     * @param {string} [replacement="***"] - Replacement string for filtered content
+     * @returns {string} Filtered text with inappropriate content replaced
+     * @description Processes text content in chunks, applying filtering rules
+     * to detect and replace inappropriate content. Handles large text efficiently
+     * by breaking it into manageable chunks.
+     *
+     * @example
+     * ```typescript
+     * const filterManager = ContentFilterManager.getInstance();
+     * const filtered = filterManager.filterText('text to filter', '***');
+     * ```
+     *
+     * @throws {TypeError} If text is not a string
+     */
+    filterText(text: string, replacement?: string): string;
+    /**
+     * Applies the actual filtering logic to a single chunk.
+     * @private
+     * @param {string} chunk - Text chunk to filter
+     * @param {string} replacement - Replacement string for filtered content
+     * @returns {string} Filtered text chunk
+     * @description Applies filtering rules to a single chunk of text,
+     * replacing inappropriate content with the specified replacement string.
+     *
+     * @throws {TypeError} If chunk is not a string
+     */
+    private applyFilters;
+}
+/**
+ * Main scraping function that processes and filters web content.
+ * @param {string} url - The URL to scrape
+ * @returns {Promise<{flaggedDomain: boolean, containsCensored: boolean, filteredTexts: string[]} | {error: string}>}
+ * Object containing scraping results or error information
+ * @description Coordinates the entire scraping process including:
+ * - URL validation
+ * - Browser management
+ * - Content extraction
+ * - Text processing
+ * - Content filtering
+ * @throws {Error} Various errors related to browser operations or content processing
+ */
+declare function scrape(url: string, browser?: Browser | null): Promise<{
+    flaggedDomain?: boolean;
+    containsCensored?: boolean;
+    filteredTexts?: string[];
+    error?: string;
+}>;
+/**
+ * Initializes the content filtering system.
+ * @description Creates the singleton instance of ContentFilterManager and
+ * loads all filtering data structures
+ */
+declare const initializeFilterWords: () => void;
+/**
+ * Filters text content using the ContentFilterManager.
+ * @param {string} text - The text to filter
+ * @param {string} [replace="***"] - The replacement string for filtered content
+ * @returns {string} The filtered text with inappropriate content replaced
+ * @description Provides a convenient wrapper around ContentFilterManager's filterText method
+ */
+declare const filterText: (text: string, replace?: string) => string;
+
+/**
+ * @fileoverview Advanced web scraping and content processing system that provides comprehensive functionality
+ * for recursive web crawling, content extraction, screenshot capture, and AI-powered content analysis.
+ *
+ * @module web
+ * @requires playwright - For browser automation and screenshot capture
+ * @requires node:path - For file path handling
+ * @requires node:fs/promises - For async file operations
+ * @requires ./scraper.js - Content extraction and filtering logic
+ * @requires ./content-analyzer.js - Content analysis and prompt generation
+ * @requires ../constants/gemini-settings.js - Configuration for Gemini LLM
+ * @requires ./content-filter.js - Content filtering and moderation
+ *
+ * @description
+ * This module implements a sophisticated web scraping and content processing system with the following key capabilities:
+ *
+ * - Multi-threaded web scraping using a thread pool for concurrent processing
+ * - Recursive crawling of websites while respecting domain boundaries
+ * - Automated screenshot capture at different scroll positions
+ * - Content extraction and filtering using custom scraping logic
+ * - AI-powered content analysis and structuring using Google's Gemini LLM
+ * - File-based storage of raw and processed content with organized directory structure
+ * - Error handling and recovery mechanisms
+ * - Content moderation and NSFW filtering
+ * - Dynamic prompt generation based on content analysis
+ *
+ * The system is designed to be highly scalable and configurable while maintaining clean separation of concerns
+ * between different processing stages. It uses a modular architecture with specialized components for:
+ *
+ * - Browser automation (Playwright)
+ * - Content extraction (Scraper)
+ * - Content filtering (ContentFilterManager)
+ * - Content analysis (ContentAnalyzer)
+ * - Prompt generation (PromptGenerator)
+ * - File system operations
+ *
+ * Key Features:
+ * - Configurable output directory structure
+ * - Automatic handling of relative/absolute URLs
+ * - Intelligent URL deduplication
+ * - Robust error handling and recovery
+ * - Modular design for easy extension
+ * - Comprehensive logging and debugging
+ * - Memory efficient processing
+ * - Rate limiting and throttling support
+ * - Configurable content filtering
+ * - AI-powered content analysis
+ *
+ * Processing Flow:
+ * 1. URL validation and normalization
+ * 2. Browser initialization with optimized settings
+ * 3. Page load and screenshot capture
+ * 4. Content extraction and initial filtering
+ * 5. NSFW/content moderation checks
+ * 6. AI-powered content analysis
+ * 7. File storage with organized structure
+ * 8. Link discovery and recursive processing
+ * 9. Error handling and recovery
+ * 10. Resource cleanup
+ *
+ * Configuration Options:
+ * - Output directory structure
+ * - Browser launch parameters
+ * - Content filtering rules
+ * - AI model settings
+ * - Rate limiting parameters
+ * - Domain boundaries
+ * - File naming conventions
+ * - Screenshot settings
+ *
+ * Error Handling:
+ * - Network failures
+ * - Invalid URLs
+ * - Content extraction errors
+ * - AI processing failures
+ * - File system errors
+ * - Memory constraints
+ * - Timeout conditions
+ *
+ * Performance Considerations:
+ * - Memory usage optimization
+ * - Concurrent processing limits
+ * - Resource cleanup
+ * - Caching strategies
+ * - Network efficiency
+ * - Storage optimization
+ *
+ * Security Features:
+ * - NSFW content filtering
+ * - Domain validation
+ * - Content sanitization
+ * - Resource limits
+ * - Safe file handling
+ *
+ * @example
+ * ```typescript
+ * // Initialize scraper with custom output directory
+ * const scraper = new WebScraper("custom_output");
+ *
+ * // Configure content filter
+ * scraper.contentFilter.setRules({
+ *   maxLength: 10000,
+ *   allowedDomains: ['example.com'],
+ *   blockedKeywords: ['spam', 'adult']
+ * });
+ *
+ * try {
+ *   // Start recursive scraping
+ *   const results = await scraper.scrapeWebsite("https://example.com");
+ *
+ *   // Process results
+ *   for (const [url, result] of results) {
+ *     if (result.error) {
+ *       console.error(`Error processing ${url}:`, result.error);
+ *       continue;
+ *     }
+ *
+ *     // Access processed content
+ *     const content = await fs.readFile(result.processedContentPath, 'utf-8');
+ *     console.log(`Processed ${url}:`, {
+ *       rawContent: result.contentPath,
+ *       processedContent: result.processedContentPath,
+ *       screenshot: result.screenshot,
+ *       timestamp: new Date(result.timestamp)
+ *     });
+ *   }
+ * } catch (error) {
+ *   console.error("Scraping failed:", error);
+ * }
+ * ```
+ *
+ * @see {@link PageResult} for details on processing results
+ * @see {@link ContentFilterManager} for content filtering capabilities
+ * @see {@link ContentAnalyzer} for AI analysis features
+ * @see {@link PromptGenerator} for dynamic prompt generation
+ *
+ * @license MIT
+ * @author Original author and contributors
+ * @version 1.0.0
+ * @since 1.0.0
+ * @copyright 2024
+ */
+
+/**
+ * Represents the complete result of processing a single web page, including all generated artifacts
+ * and metadata.
+ *
+ * @interface PageResult
+ * @property {string} url - The fully qualified URL of the processed web page
+ * @property {string} contentPath - Filesystem path to the raw scraped content file
+ * @property {string} processedContentPath - Filesystem path to the AI-processed and structured content file
+ * @property {string} screenshot - Filesystem path to the captured page screenshot
+ * @property {string} [error] - Optional error message if any stage of processing failed
+ * @property {number} timestamp - Unix timestamp (in milliseconds) when processing completed
+ *
+ * The PageResult interface provides a comprehensive record of all artifacts and metadata
+ * generated during the processing of a single web page. This includes:
+ *
+ * - Original URL for reference and deduplication
+ * - Paths to both raw and processed content files
+ * - Screenshot location for visual reference
+ * - Error information if processing failed
+ * - Timestamp for tracking and ordering
+ *
+ * Use Cases:
+ * - Tracking processing status and results
+ * - Error handling and recovery
+ * - Content access and retrieval
+ * - Processing verification
+ * - Audit trail
+ *
+ * @example
+ * ```typescript
+ * // Successful processing result
+ * const successResult: PageResult = {
+ *   url: 'https://example.com/page',
+ *   contentPath: 'output/content/example_com_page_1234567890.txt',
+ *   processedContentPath: 'output/processed/example_com_page_1234567890.txt',
+ *   screenshot: 'output/screenshots/example_com_page_0.png',
+ *   timestamp: Date.now()
+ * };
+ *
+ * // Failed processing result
+ * const errorResult: PageResult = {
+ *   url: 'https://example.com/invalid',
+ *   contentPath: '',
+ *   processedContentPath: '',
+ *   screenshot: '',
+ *   error: 'Failed to load page: 404 Not Found',
+ *   timestamp: Date.now()
+ * };
+ * ```
+ */
+interface PageResult {
+    url: string;
+    contentPath: string;
+    processedContentPath: string;
+    screenshot: string;
+    error?: string;
+    timestamp: number;
+}
+/**
+ * Core class implementing the web scraping and content processing system. Handles all aspects
+ * of the scraping process from URL discovery to content storage.
+ *
+ * @class WebScraper
+ *
+ * @property {Browser | null} browser - Playwright browser instance used for automation
+ * @property {Map<string, PageResult>} results - Map storing processing results for each URL
+ * @property {Set<string>} processedUrls - Set of URLs that have been processed to prevent duplicates
+ * @property {string} outputDir - Root directory for storing all generated files and artifacts
+ * @property {ContentFilterManager} contentFilter - Instance of content filtering manager
+ * @property {string} baseUrl - Base URL/domain for the current scraping session
+ *
+ * Key Responsibilities:
+ * 1. Browser Management
+ *    - Initialization with optimized settings
+ *    - Resource cleanup
+ *    - Error handling
+ *
+ * 2. Content Processing
+ *    - URL validation and normalization
+ *    - Content extraction
+ *    - Screenshot capture
+ *    - AI analysis
+ *    - Content filtering
+ *
+ * 3. File Management
+ *    - Directory structure creation
+ *    - File naming and organization
+ *    - Content storage
+ *    - Resource cleanup
+ *
+ * 4. URL Management
+ *    - Deduplication
+ *    - Domain boundary enforcement
+ *    - Link discovery
+ *    - Queue management
+ *
+ * 5. Error Handling
+ *    - Network failures
+ *    - Content processing errors
+ *    - Resource constraints
+ *    - Recovery mechanisms
+ *
+ * Processing Stages:
+ * 1. Initialization
+ *    - Directory setup
+ *    - Browser launch
+ *    - Filter configuration
+ *
+ * 2. URL Processing
+ *    - Validation
+ *    - Deduplication
+ *    - Domain checking
+ *
+ * 3. Content Extraction
+ *    - Page loading
+ *    - Screenshot capture
+ *    - Content scraping
+ *
+ * 4. Content Processing
+ *    - Filtering
+ *    - AI analysis
+ *    - Structure generation
+ *
+ * 5. Storage
+ *    - File organization
+ *    - Content saving
+ *    - Metadata tracking
+ *
+ * 6. Link Discovery
+ *    - URL extraction
+ *    - Validation
+ *    - Queue management
+ *
+ * 7. Cleanup
+ *    - Resource release
+ *    - Error handling
+ *    - Status reporting
+ *
+ * @example
+ * ```typescript
+ * // Initialize scraper with custom settings
+ * const scraper = new WebScraper("output_dir");
+ *
+ * try {
+ *   // Configure content filter
+ *   scraper.contentFilter.setRules({
+ *     maxLength: 50000,
+ *     allowedDomains: ['example.com']
+ *   });
+ *
+ *   // Start recursive scraping
+ *   const results = await scraper.scrapeWebsite("https://example.com");
+ *
+ *   // Process results
+ *   for (const [url, result] of results) {
+ *     if (result.error) {
+ *       console.error(`Error processing ${url}:`, result.error);
+ *       continue;
+ *     }
+ *
+ *     // Access processed content
+ *     const content = await fs.readFile(result.processedContentPath, 'utf-8');
+ *     console.log(`Successfully processed ${url}`);
+ *   }
+ * } catch (error) {
+ *   console.error("Scraping failed:", error);
+ * }
+ * ```
+ *
+ * @throws {Error} Invalid URL provided
+ * @throws {Error} Browser initialization failed
+ * @throws {Error} Content processing failed
+ * @throws {Error} File system operation failed
+ */
+declare class WebScraper {
+    private browser;
+    private results;
+    private processedUrls;
+    private outputDir;
+    readonly contentFilter: ContentFilterManager;
+    private baseUrl;
+    private sentimentAnalyzer;
+    /**
+     * Creates a new WebScraper instance.
+     *
+     * @param {string} outputDir - Directory where scraped content and artifacts will be stored
+     * @default "scraping_output"
+     *
+     * The constructor initializes a new WebScraper instance with the following setup:
+     *
+     * 1. Output Directory
+     *    - Creates base directory for all artifacts
+     *    - Organizes subdirectories for different content types
+     *    - Handles path normalization
+     *
+     * 2. Content Filter
+     *    - Initializes content filtering system
+     *    - Sets up default filtering rules
+     *    - Prepares moderation capabilities
+     *
+     * Directory Structure:
+     * ```
+     * outputDir/
+     * ├── content/         # Raw scraped content
+     * │   └── [domain]/    # Organized by domain
+     * ├── processed/       # AI-processed content
+     * │   └── [domain]/    # Organized by domain
+     * └── screenshots/     # Page screenshots
+     *     └── [domain]/    # Organized by domain
+     * ```
+     *
+     * @example
+     * ```typescript
+     * // Basic initialization
+     * const scraper = new WebScraper();
+     *
+     * // Custom output directory
+     * const customScraper = new WebScraper("custom/output/path");
+     * ```
+     *
+     * @throws {Error} If directory creation fails
+     * @throws {Error} If content filter initialization fails
+     */
+    constructor(outputDir?: string);
+    /**
+     * Main entry point for scraping a website. Initializes the browser, processes the starting URL,
+     * and recursively crawls linked pages within the same domain.
+     *
+     * Processing Flow:
+     * 1. URL Validation
+     *    - Format checking
+     *    - Domain extraction
+     *    - Protocol verification
+     *
+     * 2. Environment Setup
+     *    - Directory initialization
+     *    - Browser launch
+     *    - Resource allocation
+     *
+     * 3. Content Processing
+     *    - Page loading
+     *    - Content extraction
+     *    - Screenshot capture
+     *    - AI analysis
+     *
+     * 4. Link Discovery
+     *    - URL extraction
+     *    - Domain filtering
+     *    - Queue management
+     *
+     * 5. Resource Management
+     *    - Memory monitoring
+     *    - Connection handling
+     *    - Cleanup operations
+     *
+     * Error Handling:
+     * - Invalid URLs
+     * - Network failures
+     * - Browser crashes
+     * - Memory constraints
+     * - Timeout conditions
+     *
+     * @param {string} url - Starting URL to begin scraping from
+     * @returns {Promise<Map<string, PageResult>>} Map of results for all processed URLs
+     * @throws {Error} If URL is invalid or scraping fails
+     *
+     * @example
+     * ```typescript
+     * const scraper = new WebScraper("output");
+     *
+     * try {
+     *   // Start scraping
+     *   const results = await scraper.scrapeWebsite("https://example.com");
+     *
+     *   // Process successful results
+     *   for (const [url, result] of results) {
+     *     if (!result.error) {
+     *       console.log(`Successfully processed ${url}`);
+     *       console.log(`Content saved to: ${result.processedContentPath}`);
+     *       console.log(`Screenshot saved to: ${result.screenshot}`);
+     *     }
+     *   }
+     *
+     *   // Handle errors
+     *   const errors = Array.from(results.entries())
+     *     .filter(([_, result]) => result.error)
+     *     .map(([url, result]) => ({url, error: result.error}));
+     *
+     *   if (errors.length > 0) {
+     *     console.error("Encountered errors:", errors);
+     *   }
+     * } catch (error) {
+     *   console.error("Fatal error during scraping:", error);
+     * }
+     * ```
+     */
+    scrapeWebsite(url: string): Promise<Map<string, PageResult>>;
+    /**
+     * Creates required output directories if they don't exist.
+     *
+     * Directory Structure:
+     * ```
+     * outputDir/
+     * ├── content/         # Raw scraped content
+     * │   └── [domain]/    # Organized by domain
+     * ├── processed/       # AI-processed content
+     * │   └── [domain]/    # Organized by domain
+     * └── screenshots/     # Page screenshots
+     *     └── [domain]/    # Organized by domain
+     * ```
+     *
+     * @private
+     * @returns {Promise<void>}
+     *
+     * @throws {Error} If directory creation fails
+     * @throws {Error} If permissions are insufficient
+     * @throws {Error} If disk space is insufficient
+     */
+    private initializeDirectories;
+    /**
+     * Processes a single web page, extracting content, capturing a screenshot, and analyzing content.
+     * Also discovers and processes linked pages within the same domain.
+     *
+     * Processing Stages:
+     * 1. URL Validation
+     *    - Format checking
+     *    - Deduplication
+     *    - Content type verification
+     *
+     * 2. Content Safety
+     *    - Domain checking
+     *    - NSFW detection
+     *    - Content moderation
+     *
+     * 3. Page Processing
+     *    - Loading and rendering
+     *    - Screenshot capture
+     *    - Content extraction
+     *
+     * 4. Content Analysis
+     *    - Text filtering
+     *    - AI processing
+     *    - Structure generation
+     *
+     * 5. Link Discovery
+     *    - URL extraction
+     *    - Domain filtering
+     *    - Queue management
+     *
+     * Error Handling:
+     * - Network failures
+     * - Timeout conditions
+     * - Content extraction errors
+     * - Processing failures
+     * - Resource constraints
+     *
+     * @private
+     * @param {string} url - URL of the page to process
+     * @returns {Promise<PageResult>} Processing result for the page
+     *
+     * @example
+     * ```typescript
+     * try {
+     *   const result = await scraper.processSinglePage("https://example.com/page");
+     *
+     *   if (result.error) {
+     *     console.error(`Processing failed: ${result.error}`);
+     *     return;
+     *   }
+     *
+     *   // Access results
+     *   console.log("Raw content:", result.contentPath);
+     *   console.log("Processed content:", result.processedContentPath);
+     *   console.log("Screenshot:", result.screenshot);
+     *   console.log("Processed at:", new Date(result.timestamp));
+     * } catch (error) {
+     *   console.error("Fatal error:", error);
+     * }
+     * ```
+     *
+     * @throws {Error} If page loading fails
+     * @throws {Error} If content extraction fails
+     * @throws {Error} If processing fails
+     */
+    private processSinglePage;
+    /**
+     * Processes extracted content using Google's Gemini LLM for analysis and structuring.
+     *
+     * Processing Steps:
+     * 1. Content Preparation
+     *    - Text filtering
+     *    - Format validation
+     *    - Length checking
+     *
+     * 2. Context Analysis
+     *    - URL analysis
+     *    - Content type detection
+     *    - Structure identification
+     *
+     * 3. Prompt Generation
+     *    - Dynamic template selection
+     *    - Context integration
+     *    - Parameter optimization
+     *
+     * 4. AI Processing
+     *    - Model selection
+     *    - Safety settings
+     *    - Response handling
+     *
+     * Error Handling:
+     * - Content validation
+     * - Model errors
+     * - Timeout conditions
+     * - Response validation
+     *
+     * @private
+     * @param {string} content - Raw content to process
+     * @param {string} url - URL of the content source
+     * @returns {Promise<string>} Processed and structured content
+     *
+     * @throws {Error} If content is invalid
+     * @throws {Error} If LLM processing fails
+     * @throws {Error} If response is invalid
+     *
+     * @example
+     * ```typescript
+     * try {
+     *   const rawContent = "Example raw content...";
+     *   const url = "https://example.com";
+     *
+     *   const processed = await scraper.processWithLLM(rawContent, url);
+     *   console.log("Processed content:", processed);
+     * } catch (error) {
+     *   console.error("Processing failed:", error);
+     * }
+     * ```
+     */
+    private processWithLLM;
+    /**
+     * Takes a full page screenshot of the current page
+     *
+     * Screenshot Process:
+     * 1. Page Preparation
+     *    - Viewport setup
+     *    - Content loading
+     *    - Animation completion
+     *
+     * 2. Capture Settings
+     *    - Full page mode
+     *    - Resolution configuration
+     *    - Format selection
+     *
+     * 3. File Management
+     *    - Path generation
+     *    - Directory creation
+     *    - File saving
+     *
+     * Error Handling:
+     * - Page loading issues
+     * - Screenshot failures
+     * - Storage errors
+     *
+     * @private
+     * @param {Page} page - Playwright page instance
+     * @param {string} url - URL being captured
+     * @returns {Promise<string>} Path to saved screenshot
+     *
+     * @throws {Error} If screenshot capture fails
+     * @throws {Error} If file saving fails
+     *
+     * @example
+     * ```typescript
+     * const page = await browser.newPage();
+     * await page.goto(url);
+     *
+     * try {
+     *   const screenshotPath = await scraper.takeScreenshot(page, url);
+     *   console.log("Screenshot saved to:", screenshotPath);
+     * } catch (error) {
+     *   console.error("Screenshot capture failed:", error);
+     * }
+     * ```
+     */
+    private takeScreenshot;
+    /**
+     * Saves content to a file with organized directory structure based on URL path.
+     *
+     * File Organization:
+     * 1. Path Generation
+     *    - URL parsing
+     *    - Path cleaning
+     *    - Directory structure
+     *
+     * 2. Content Validation
+     *    - File type checking
+     *    - Content verification
+     *    - Size limits
+     *
+     * 3. Directory Management
+     *    - Path creation
+     *    - Permissions
+     *    - Existing files
+     *
+     * 4. File Operations
+     *    - Content writing
+     *    - Atomic saves
+     *    - Cleanup
+     *
+     * Directory Structure:
+     * ```
+     * outputDir/
+     * └── [domain]/
+     *     ├── content/
+     *     │   └── [path]/
+     *     │       └── content-[timestamp].txt
+     *     ├── processed/
+     *     │   └── [path]/
+     *     │       └── processed-[timestamp].txt
+     *     └── screenshots/
+     *         └── [path]/
+     *             └── screenshot-[timestamp].png
+     * ```
+     *
+     * @private
+     * @param {string} content - Content to save
+     * @param {'content' | 'processed' | 'screenshots'} type - Type of content being saved
+     * @param {string} url - Source URL
+     * @param {string} [fileExtension='.txt'] - File extension to use
+     * @returns {Promise<string>} Path to saved file
+     *
+     * @throws {Error} If file is non-textual
+     * @throws {Error} If saving fails
+     * @throws {Error} If directory creation fails
+     *
+     * @example
+     * ```typescript
+     * try {
+     *   // Save raw content
+     *   const contentPath = await scraper.saveToFile(
+     *     "Raw content...",
+     *     "content",
+     *     "https://example.com/page"
+     *   );
+     *
+     *   // Save processed content
+     *   const processedPath = await scraper.saveToFile(
+     *     "Processed content...",
+     *     "processed",
+     *     "https://example.com/page"
+     *   );
+     *
+     *   console.log("Content saved to:", contentPath);
+     *   console.log("Processed content saved to:", processedPath);
+     * } catch (error) {
+     * @throws {Error} If file is non-textual or saving fails
+     */
+    private saveToFile;
+    /**
+     * Validates AI generated content for safety and sentiment
+     * @private
+     * @param {string} content - AI generated content to validate
+     * @returns {Promise<{isValid: boolean, reason?: string}>}
+     */
+    private validateAIResponse;
+    /**
+     * Process AI response with safety checks
+     * @private
+     * @param {string} aiResponse - Response from AI model
+     * @returns {Promise<string>} Validated and processed response
+     * @throws {Error} If content validation fails
+     */
+    private processAIResponse;
+}
+
+/**
+ * @fileoverview Content analysis and prompt generation system for web content processing
+ * @module content-analyzer
+ * @description Provides comprehensive functionality for analyzing web content structure and generating
+ * context-aware prompts for LLM processing. The module includes two main classes:
+ * - ContentAnalyzer: Analyzes web content to determine its context and characteristics
+ * - PromptGenerator: Generates tailored prompts based on the analyzed content context
+ *
+ * Key features:
+ * - URL pattern matching and content signal detection
+ * - Content type classification (article, product, profile, etc.)
+ * - Structure analysis (narrative, analytical, technical, etc.)
+ * - Context-aware prompt generation
+ * - Flexible template system for different content types
+ *
+ * @example
+ * ```typescript
+ * // Analyze content
+ * const context = ContentAnalyzer.analyzeContent(url, htmlContent);
+ *
+ * // Generate appropriate prompt
+ * const prompt = PromptGenerator.generatePrompt(context, content);
+ * ```
+ */
+/**
+ * Represents the context and characteristics of analyzed content
+ * @interface ContentContext
+ * @property {('article'|'product'|'category'|'profile'|'general')} pageType - The type of page being analyzed:
+ *   - article: Blog posts, news articles, editorial content
+ *   - product: Product pages, item listings, shop entries
+ *   - category: Category pages, department listings, sections
+ *   - profile: User profiles, about pages, portfolios
+ *   - general: Default type for unclassified content
+ * @property {('brief'|'standard'|'detailed')} contentLength - The relative length/depth of the content:
+ *   - brief: Short-form content, summaries
+ *   - standard: Medium-length content
+ *   - detailed: Long-form, in-depth content
+ * @property {('narrative'|'analytical'|'technical'|'descriptive')} structureType - The structural style of the content:
+ *   - narrative: Story-based, chronological flow
+ *   - analytical: Data-driven, research-oriented
+ *   - technical: Specification-focused, procedural
+ *   - descriptive: Feature-focused, explanatory
+ * @property {('general'|'technical'|'business'|'academic')} targetAudience - The intended audience for the content:
+ *   - general: General public, non-specialized readers
+ *   - technical: Technical professionals, developers
+ *   - business: Business professionals, stakeholders
+ *   - academic: Researchers, students, educators
+ */
+interface ContentContext$1 {
+    pageType: 'article' | 'product' | 'category' | 'profile' | 'general';
+    contentLength: 'brief' | 'standard' | 'detailed';
+    structureType: 'narrative' | 'analytical' | 'technical' | 'descriptive';
+    targetAudience: 'general' | 'technical' | 'business' | 'academic';
+}
+/**
+ * Analyzes web content to determine its context and characteristics
+ * @class ContentAnalyzer
+ * @static
+ * @description Provides static methods for analyzing web content and determining its context.
+ * Uses pattern matching and content signals to classify content and determine appropriate
+ * processing strategies. The analysis considers:
+ * - URL patterns and structure
+ * - Content keywords and signals
+ * - Document structure and elements
+ * - Content indicators and metadata
+ */
+declare class ContentAnalyzer {
+    /**
+     * Predefined patterns for analyzing different types of content routes
+     * @private
+     * @static
+     * @readonly
+     * @type {RouteAnalysis[]}
+     * @description Array of route analysis configurations that define:
+     * - URL patterns to match different content types
+     * - Content signals associated with each type
+     * - Default context settings for matched content
+     * Each configuration targets a specific content category (article, product, etc.)
+     * and provides the basis for content classification.
+     */
+    private static readonly routePatterns;
+    /**
+     * Extracts content signals from HTML content by analyzing structure and keywords
+     * @private
+     * @static
+     * @param {string} content - The HTML content to analyze
+     * @returns {Set<string>} Set of identified content signals
+     * @description Analyzes HTML content to identify structural and keyword signals:
+     * - Checks for presence of headers, lists, and tables
+     * - Identifies content-specific keywords and patterns
+     * - Detects pricing information and author attribution
+     * - Recognizes profile and biographical content
+     * The signals are used to help classify and contextualize the content.
+     */
+    private static getContentSignals;
+    /**
+     * Analyzes content and URL to determine the appropriate content context
+     * @public
+     * @static
+     * @param {string} url - The URL of the content being analyzed
+     * @param {string} content - The HTML content to analyze
+     * @returns {ContentContext} The determined content context
+     * @description Performs comprehensive content analysis by:
+     * 1. Extracting and analyzing the URL path
+     * 2. Identifying content signals from the HTML
+     * 3. Matching against predefined route patterns
+     * 4. Determining the most appropriate content context
+     * If no specific matches are found, returns a default general context.
+     *
+     * @example
+     * ```typescript
+     * const context = ContentAnalyzer.analyzeContent(
+     *   'https://example.com/blog/article-1',
+     *   '<html>...</html>'
+     * );
+     * ```
+     */
+    static analyzeContent(url: string, content: string): ContentContext$1;
+}
+/**
+ * Generates context-aware prompts for LLM content processing
+ * @class PromptGenerator
+ * @static
+ * @description Provides functionality for generating tailored prompts based on content context.
+ * Features:
+ * - Template-based prompt generation
+ * - Context-aware prompt customization
+ * - Support for multiple content types and structures
+ * - Fallback to default prompts when needed
+ */
+declare class PromptGenerator {
+    /**
+     * Template definitions for different content types and structures
+     * @private
+     * @static
+     * @readonly
+     * @type {Object}
+     * @description Comprehensive template system that provides:
+     * - Content type-specific templates (article, product, profile)
+     * - Structure-specific variations (narrative, analytical, technical)
+     * - Detailed processing instructions
+     * - Placeholder support for content insertion
+     * Templates are organized hierarchically by content type and structure.
+     */
+    private static readonly promptTemplates;
+    /**
+     * Generates an appropriate prompt based on content context
+     * @public
+     * @static
+     * @param {ContentContext} context - The analyzed content context
+     * @param {string} content - The content to be processed
+     * @returns {string} Generated prompt for LLM processing
+     * @description Generates a context-appropriate prompt by:
+     * 1. Selecting appropriate template based on content type
+     * 2. Choosing structure-specific variation
+     * 3. Inserting content into template
+     * 4. Falling back to default prompt if no specific template exists
+     *
+     * @example
+     * ```typescript
+     * const prompt = PromptGenerator.generatePrompt(
+     *   { pageType: 'article', structureType: 'narrative', ... },
+     *   'Article content...'
+     * );
+     * ```
+     */
+    static generatePrompt(context: ContentContext$1, content: string): string;
+    /**
+     * Provides a default prompt when no specific template matches
+     * @private
+     * @static
+     * @param {string} content - The content to be processed
+     * @returns {string} Default analysis prompt
+     * @description Generates a generic but comprehensive prompt for content analysis
+     * when no specific template matches the content context. The default prompt
+     * focuses on:
+     * - Topic extraction
+     * - Content organization
+     * - Redundancy removal
+     * - Clarity and readability
+     */
+    private static getDefaultPrompt;
+}
+
+interface WorkerMessage<T = any> {
+    id: string;
+    type: "TASK" | "RESULT" | "ERROR" | "STATUS" | "READY" | "INIT";
+    payload: T;
+    timestamp: number;
+}
+interface WorkerTask {
+    id: string;
+    type: "TASK" | "RESULT" | "ERROR" | "STATUS" | "READY" | "INIT";
+    url?: string;
+    data?: any;
+}
+interface TrieNode {
+    children: {
+        [key: string]: TrieNode;
+    };
+    isEndOfWord: boolean;
+}
+interface ContentContext {
+    pageType: 'article' | 'product' | 'category' | 'profile' | 'general';
+    contentLength: 'brief' | 'standard' | 'detailed';
+    structureType: 'narrative' | 'analytical' | 'technical' | 'descriptive';
+    targetAudience: 'general' | 'technical' | 'business' | 'academic';
+}
+interface RouteAnalysis {
+    patterns: RegExp[];
+    signals: string[];
+    context: ContentContext;
+}
+interface CodeBlock {
+    language: string;
+    code: string;
+    lineNumbers?: boolean;
+}
+
+declare const nsfwNames: Readonly<Record<string, string>>;
+type NsfwName = keyof typeof nsfwNames;
+
+declare const nsfw: Readonly<Record<string, string>>;
+
+declare const robots: Readonly<Record<string, string>>;
+type Robot = keyof typeof robots;
+
+declare const slurs: Readonly<Record<string, string>>;
+type Slur = keyof typeof slurs;
+
+/**
+ * @fileoverview Configuration settings for Google's Gemini AI model integration
+ * @module gemini-settings
+ * @description Provides configuration constants and settings for interacting with the Gemini AI API,
+ * including model selection, API authentication, and content safety thresholds
+ */
+
+/**
+ * The specific Gemini model version to use
+ * @constant {string}
+ * @description Specifies the Gemini 1.5 Flash model, optimized for fast inference
+ */
+declare const model = "gemini-1.5-flash";
+/**
+ * Google AI API key loaded from environment variables
+ * @constant {string}
+ * @description API key for authenticating with Google's AI services. Falls back to empty string if not configured
+ */
+declare const API_KEY: string;
+/**
+ * Initialized Google Generative AI client
+ * @constant {GoogleGenerativeAI}
+ * @description Main client instance for interacting with Gemini AI services
+ */
+declare const genAI: GoogleGenerativeAI;
+/**
+ * Generation configuration settings
+ * @constant {undefined}
+ * @description Currently undefined, can be used to specify generation parameters like temperature, top-k, etc.
+ */
+declare const generationConfig: undefined;
+/**
+ * Content safety threshold settings
+ * @constant {Array<{category: HarmCategory, threshold: HarmBlockThreshold}>}
+ * @description Configures content filtering thresholds for different harm categories:
+ * - Harassment
+ * - Hate Speech
+ * - Sexually Explicit Content
+ * - Dangerous Content
+ * All thresholds are currently set to BLOCK_NONE for maximum permissiveness
+ */
+declare const safetySettings: {
+    category: HarmCategory;
+    threshold: HarmBlockThreshold;
+}[];
+
+export { API_KEY, type CodeBlock, ContentAnalyzer, type ContentContext, ContentFilterManager, type NsfwName, PromptGenerator, type Robot, type RouteAnalysis, type Slur, type TrieNode, WebScraper, type WorkerMessage, type WorkerTask, filterText, genAI, generationConfig, initializeFilterWords, model, nsfw, nsfwNames, robots, safetySettings, scrape, slurs };