@humbletoes/google-search 1.0.0

package/dist/cache.js

@@ -0,0 +1,111 @@
+/**
+ * Simple LRU cache implementation for search results
+ * Reduces redundant searches and improves performance
+ */
+import logger from "./logger.js";
+export class SearchCache {
+    constructor(maxSize = 100, ttl = 300000) {
+        this.totalHits = 0;
+        this.totalMisses = 0;
+        this.cache = new Map();
+        this.maxSize = maxSize;
+        this.ttl = ttl;
+    }
+    /**
+     * Generate cache key from query and options
+     */
+    generateKey(query, limit) {
+        return `${query.toLowerCase().trim()}:${limit || 10}`;
+    }
+    /**
+      * Get cached result if valid
+      */
+    get(query, limit, ttl) {
+        const key = this.generateKey(query, limit);
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.totalMisses++;
+            logger.debug({ query }, "Cache miss");
+            return null;
+        }
+        // Check if entry is expired
+        const age = Date.now() - entry.timestamp;
+        const effectiveTtl = ttl || this.ttl;
+        if (age > effectiveTtl) {
+            this.totalMisses++;
+            logger.debug({ query, age, effectiveTtl }, "Cache entry expired");
+            this.cache.delete(key);
+            return null;
+        }
+        // Update hit count
+        entry.hits++;
+        this.totalHits++;
+        logger.info({ query, age, hits: entry.hits }, "Cache hit");
+        return entry.data;
+    }
+    /**
+      * Store result in cache
+      */
+    set(query, data, limit, ttl) {
+        const key = this.generateKey(query, limit);
+        // If cache is full, remove least recently used entry
+        if (this.cache.size >= this.maxSize && !this.cache.has(key)) {
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey) {
+                this.cache.delete(firstKey);
+                logger.debug({ removedKey: firstKey }, "Cache eviction");
+            }
+        }
+        this.cache.set(key, {
+            data,
+            timestamp: Date.now(),
+            hits: 0,
+        });
+        logger.debug({ query, cacheSize: this.cache.size }, "Cached search result");
+    }
+    /**
+     * Clear all cache entries
+     */
+    clear() {
+        const size = this.cache.size;
+        this.cache.clear();
+        logger.info({ clearedEntries: size }, "Cache cleared");
+    }
+    /**
+     * Remove expired entries
+     */
+    cleanup() {
+        const now = Date.now();
+        let removed = 0;
+        for (const [key, entry] of this.cache.entries()) {
+            if (now - entry.timestamp > this.ttl) {
+                this.cache.delete(key);
+                removed++;
+            }
+        }
+        if (removed > 0) {
+            logger.info({ removed, remaining: this.cache.size }, "Cache cleanup completed");
+        }
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const now = Date.now();
+        const entries = Array.from(this.cache.entries()).map(([key, entry]) => ({
+            key,
+            age: now - entry.timestamp,
+            hits: entry.hits,
+        }));
+        return {
+            size: this.cache.size,
+            maxSize: this.maxSize,
+            ttl: this.ttl,
+            hits: this.totalHits,
+            misses: this.totalMisses,
+            entries,
+        };
+    }
+}
+// Note: No singleton instance, create per use with desired TTL
+//# sourceMappingURL=cache.js.map

package/dist/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,MAAM,MAAM,aAAa,CAAC;AAQjC,MAAM,OAAO,WAAW;IAOtB,YAAY,UAAkB,GAAG,EAAE,MAAc,MAAM;QAH/C,cAAS,GAAW,CAAC,CAAC;QACtB,gBAAW,GAAW,CAAC,CAAC;QAG9B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAa,EAAE,KAAc;QAC/C,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;IACxD,CAAC;IAED;;QAEI;IACH,GAAG,CAAC,KAAa,EAAE,KAAc,EAAE,GAAY;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,4BAA4B;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC;QACzC,MAAM,YAAY,GAAG,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACrC,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,qBAAqB,CAAC,CAAC;YAClE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,mBAAmB;QACnB,KAAK,CAAC,IAAI,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;QAC3D,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;QAEI;IACH,GAAG,CAAC,KAAa,EAAE,IAAoB,EAAE,KAAc,EAAE,GAAY;QACpE,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAE3C,qDAAqD;QACrD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAChD,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,eAAe,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACrC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QAQN,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YACtE,GAAG;YACH,GAAG,EAAE,GAAG,GAAG,KAAK,CAAC,SAAS;YAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,SAAS;YACpB,MAAM,EAAE,IAAI,CAAC,WAAW;YACxB,OAAO;SACR,CAAC;IACJ,CAAC;CACF;AAED,+DAA+D"}

package/dist/errors.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Base error class for search operations
+ */
+export declare class SearchError extends Error {
+    code: string;
+    retryable: boolean;
+    constructor(message: string, code: string, retryable?: boolean);
+}
+/**
+ * Error thrown when CAPTCHA verification is required
+ */
+export declare class CaptchaError extends SearchError {
+    constructor(message: string);
+}
+/**
+ * Error thrown when network issues occur
+ */
+export declare class NetworkError extends SearchError {
+    constructor(message: string);
+}
+/**
+ * Manages retry logic for operations that may fail
+ */
+export declare class RetryManager {
+    executeWithRetry<T>(operation: () => Promise<T>, maxRetries?: number, baseDelay?: number): Promise<T>;
+}

package/dist/errors.js

@@ -0,0 +1,48 @@
+/**
+ * Base error class for search operations
+ */
+export class SearchError extends Error {
+    constructor(message, code, retryable = false) {
+        super(message);
+        this.code = code;
+        this.retryable = retryable;
+        this.name = 'SearchError';
+    }
+}
+/**
+ * Error thrown when CAPTCHA verification is required
+ */
+export class CaptchaError extends SearchError {
+    constructor(message) {
+        super(message, 'CAPTCHA_REQUIRED', true);
+    }
+}
+/**
+ * Error thrown when network issues occur
+ */
+export class NetworkError extends SearchError {
+    constructor(message) {
+        super(message, 'NETWORK_ERROR', true);
+    }
+}
+/**
+ * Manages retry logic for operations that may fail
+ */
+export class RetryManager {
+    async executeWithRetry(operation, maxRetries = 3, baseDelay = 1000) {
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                return await operation();
+            }
+            catch (error) {
+                if (attempt === maxRetries || !(error instanceof SearchError) || !error.retryable) {
+                    throw error;
+                }
+                const delay = baseDelay * Math.pow(2, attempt);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+        throw new Error('Retry logic error: should not reach here');
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAY,OAAe,EAAS,IAAY,EAAS,YAAqB,KAAK;QACjF,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAQ;QAAS,cAAS,GAAT,SAAS,CAAiB;QAEjF,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAa,SAAQ,WAAW;IAC3C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAC3C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAa,SAAQ,WAAW;IAC3C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;IACxC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAY;IACvB,KAAK,CAAC,gBAAgB,CACpB,SAA2B,EAC3B,aAAqB,CAAC,EACtB,YAAoB,IAAI;QAExB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,YAAY,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;oBAClF,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/dist/filters.d.ts

@@ -0,0 +1,48 @@
+import { SearchResult } from './types.js';
+export interface FilterOptions {
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    minSnippetLength?: number;
+    maxSnippetLength?: number;
+    onlyRichSnippets?: boolean;
+    topN?: number;
+    skipFirst?: number;
+    mustInclude?: string[];
+    mustExclude?: string[];
+    contentType?: 'all' | 'html' | 'pdf' | 'doc' | 'video' | 'image';
+    dateRange?: {
+        from?: Date;
+        to?: Date;
+    };
+    minRelevanceScore?: number;
+}
+export interface FilterStats {
+    totalResults: number;
+    filteredResults: number;
+    domainDistribution: Record<string, number>;
+    contentTypeDistribution: Record<string, number>;
+    richSnippetCount: number;
+    averageSnippetLength: number;
+    dateRange?: {
+        earliest?: string;
+        latest?: string;
+    };
+}
+export declare class ResultFilter {
+    /**
+     * Filter search results based on various criteria
+     */
+    static filter(results: SearchResult[], options: FilterOptions): SearchResult[];
+    /**
+     * Sort search results by various criteria
+     */
+    static sort(results: SearchResult[], sortBy: 'position' | 'snippetLength' | 'domain', order?: 'asc' | 'desc'): SearchResult[];
+    /**
+     * Get comprehensive statistics about filtered results
+     */
+    static getStats(results: SearchResult[]): FilterStats;
+    /**
+     * Add relevance scoring to results (basic implementation)
+     */
+    static addRelevanceScores(results: SearchResult[], query: string): SearchResult[];
+}

package/dist/filters.js

@@ -0,0 +1,192 @@
+export class ResultFilter {
+    /**
+     * Filter search results based on various criteria
+     */
+    static filter(results, options) {
+        let filtered = [...results];
+        // Domain filtering
+        if (options.includeDomains && options.includeDomains.length > 0) {
+            const domains = options.includeDomains.map(d => d.toLowerCase());
+            filtered = filtered.filter(result => result.domain && domains.some(domain => result.domain.toLowerCase().includes(domain) ||
+                domain.includes(result.domain.toLowerCase())));
+        }
+        if (options.excludeDomains && options.excludeDomains.length > 0) {
+            const domains = options.excludeDomains.map(d => d.toLowerCase());
+            filtered = filtered.filter(result => !result.domain || !domains.some(domain => result.domain.toLowerCase().includes(domain) ||
+                domain.includes(result.domain.toLowerCase())));
+        }
+        // Snippet length filtering
+        if (options.minSnippetLength !== undefined) {
+            filtered = filtered.filter(result => (result.snippetLength || 0) >= options.minSnippetLength);
+        }
+        if (options.maxSnippetLength !== undefined) {
+            filtered = filtered.filter(result => (result.snippetLength || 0) <= options.maxSnippetLength);
+        }
+        // Rich snippet filtering
+        if (options.onlyRichSnippets) {
+            filtered = filtered.filter(result => result.hasRichSnippet);
+        }
+        // Content type filtering (basic URL-based detection)
+        if (options.contentType && options.contentType !== 'all') {
+            filtered = filtered.filter(result => {
+                const url = result.link.toLowerCase();
+                switch (options.contentType) {
+                    case 'pdf':
+                        return url.endsWith('.pdf') || url.includes('.pdf?');
+                    case 'doc':
+                        return url.endsWith('.doc') || url.endsWith('.docx') ||
+                            url.endsWith('.txt') || url.includes('.doc');
+                    case 'video':
+                        return url.includes('youtube.com') || url.includes('vimeo.com') ||
+                            url.endsWith('.mp4') || url.endsWith('.avi') ||
+                            url.includes('/video/');
+                    case 'image':
+                        return url.endsWith('.jpg') || url.endsWith('.jpeg') ||
+                            url.endsWith('.png') || url.endsWith('.gif') ||
+                            url.endsWith('.webp') || url.includes('/image/');
+                    case 'html':
+                    default:
+                        return !url.endsWith('.pdf') && !url.endsWith('.doc') &&
+                            !url.endsWith('.docx') && !url.endsWith('.mp4') &&
+                            !url.endsWith('.avi') && !url.includes('youtube.com') &&
+                            !url.includes('vimeo.com');
+                }
+            });
+        }
+        // Keyword filtering
+        if (options.mustInclude && options.mustInclude.length > 0) {
+            filtered = filtered.filter(result => {
+                const text = (result.title + ' ' + result.snippet).toLowerCase();
+                return options.mustInclude.every(keyword => text.includes(keyword.toLowerCase()));
+            });
+        }
+        if (options.mustExclude && options.mustExclude.length > 0) {
+            filtered = filtered.filter(result => {
+                const text = (result.title + ' ' + result.snippet).toLowerCase();
+                return !options.mustExclude.some(keyword => text.includes(keyword.toLowerCase()));
+            });
+        }
+        // Position-based filtering
+        if (options.skipFirst && options.skipFirst > 0) {
+            filtered = filtered.slice(options.skipFirst);
+        }
+        if (options.topN && options.topN > 0) {
+            filtered = filtered.slice(0, options.topN);
+        }
+        return filtered;
+    }
+    /**
+     * Sort search results by various criteria
+     */
+    static sort(results, sortBy, order = 'asc') {
+        return [...results].sort((a, b) => {
+            let comparison = 0;
+            switch (sortBy) {
+                case 'position':
+                    comparison = (a.position || 0) - (b.position || 0);
+                    break;
+                case 'snippetLength':
+                    comparison = (a.snippetLength || 0) - (b.snippetLength || 0);
+                    break;
+                case 'domain':
+                    comparison = (a.domain || '').localeCompare(b.domain || '');
+                    break;
+            }
+            return order === 'desc' ? -comparison : comparison;
+        });
+    }
+    /**
+     * Get comprehensive statistics about filtered results
+     */
+    static getStats(results) {
+        const domainDistribution = {};
+        const contentTypeDistribution = {
+            html: 0,
+            pdf: 0,
+            doc: 0,
+            video: 0,
+            image: 0,
+            other: 0
+        };
+        let richSnippetCount = 0;
+        let totalSnippetLength = 0;
+        results.forEach(result => {
+            // Domain distribution
+            const domain = result.domain || 'unknown';
+            domainDistribution[domain] = (domainDistribution[domain] || 0) + 1;
+            // Content type distribution
+            const url = result.link.toLowerCase();
+            if (url.endsWith('.pdf') || url.includes('.pdf?')) {
+                contentTypeDistribution.pdf++;
+            }
+            else if (url.endsWith('.doc') || url.endsWith('.docx') || url.endsWith('.txt')) {
+                contentTypeDistribution.doc++;
+            }
+            else if (url.includes('youtube.com') || url.includes('vimeo.com') ||
+                url.endsWith('.mp4') || url.endsWith('.avi') || url.includes('/video/')) {
+                contentTypeDistribution.video++;
+            }
+            else if (url.endsWith('.jpg') || url.endsWith('.jpeg') || url.endsWith('.png') ||
+                url.endsWith('.gif') || url.endsWith('.webp') || url.includes('/image/')) {
+                contentTypeDistribution.image++;
+            }
+            else {
+                contentTypeDistribution.html++;
+            }
+            // Rich snippet count
+            if (result.hasRichSnippet) {
+                richSnippetCount++;
+            }
+            // Snippet length
+            totalSnippetLength += result.snippetLength || 0;
+        });
+        return {
+            totalResults: results.length,
+            filteredResults: results.length,
+            domainDistribution,
+            contentTypeDistribution,
+            richSnippetCount,
+            averageSnippetLength: results.length > 0 ? totalSnippetLength / results.length : 0,
+        };
+    }
+    /**
+     * Add relevance scoring to results (basic implementation)
+     */
+    static addRelevanceScores(results, query) {
+        const queryWords = query.toLowerCase().split(/\s+/).filter(word => word.length > 2);
+        return results.map(result => {
+            let score = 0;
+            const title = result.title.toLowerCase();
+            const snippet = result.snippet.toLowerCase();
+            // Title matches are most important
+            queryWords.forEach(word => {
+                if (title.includes(word)) {
+                    score += 10;
+                }
+                if (snippet.includes(word)) {
+                    score += 5;
+                }
+            });
+            // Position bonus (earlier results are more relevant)
+            if (result.position) {
+                score += Math.max(0, 20 - result.position);
+            }
+            // Rich snippet bonus
+            if (result.hasRichSnippet) {
+                score += 5;
+            }
+            // Domain authority bonus (basic heuristic)
+            if (result.domain) {
+                const trustedDomains = ['github.com', 'stackoverflow.com', 'wikipedia.org', 'docs.microsoft.com'];
+                if (trustedDomains.some(domain => result.domain.includes(domain))) {
+                    score += 3;
+                }
+            }
+            return {
+                ...result,
+                relevanceScore: score
+            };
+        });
+    }
+}
+//# sourceMappingURL=filters.js.map

package/dist/filters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filters.js","sourceRoot":"","sources":["../src/filters.ts"],"names":[],"mappings":"AAiCA,MAAM,OAAO,YAAY;IACvB;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,OAAuB,EAAE,OAAsB;QAC3D,IAAI,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;QAE5B,mBAAmB;QACnB,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACjE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAClC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACrC,MAAM,CAAC,MAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAO,CAAC,WAAW,EAAE,CAAC,CAC9C,CACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACjE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAClC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACvC,MAAM,CAAC,MAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAO,CAAC,WAAW,EAAE,CAAC,CAC9C,CACF,CAAC;QACJ,CAAC;QAED,2BAA2B;QAC3B,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAC3C,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAClC,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,gBAAiB,CACzD,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAC3C,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAClC,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,gBAAiB,CACzD,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC7B,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC9D,CAAC;QAED,qDAAqD;QACrD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YACzD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBAClC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtC,QAAQ,OAAO,CAAC,WAAW,EAAE,CAAC;oBAC5B,KAAK,KAAK;wBACR,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;oBACvD,KAAK,KAAK;wBACR,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;4BAC7C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACtD,KAAK,OAAO;wBACV,OAAO,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;4BACxD,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;4BAC5C,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACjC,KAAK,OAAO;wBACV,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;4BAC7C,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;4BAC5C,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBAC1D,KAAK,MAAM,CAAC;oBACZ;wBACE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;4BAC9C,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;4BAC/C,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;4BACrD,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBAClC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjE,OAAO,OAAO,CAAC,WAAY,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CACrC,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBAClC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjE,OAAO,CAAC,OAAO,CAAC,WAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CACrC,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,2BAA2B;QAC3B,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC/C,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACrC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,OAAuB,EAAE,MAA+C,EAAE,QAAwB,KAAK;QACjH,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAChC,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,UAAU;oBACb,UAAU,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;oBACnD,MAAM;gBACR,KAAK,eAAe;oBAClB,UAAU,GAAG,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC;oBAC7D,MAAM;gBACR,KAAK,QAAQ;oBACX,UAAU,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;oBAC5D,MAAM;YACV,CAAC;YAED,OAAO,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,OAAuB;QACrC,MAAM,kBAAkB,GAA2B,EAAE,CAAC;QACtD,MAAM,uBAAuB,GAA2B;YACtD,IAAI,EAAE,CAAC;YACP,GAAG,EAAE,CAAC;YACN,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC;SACT,CAAC;QAEF,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YACvB,sBAAsB;YACtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC;YAC1C,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAEnE,4BAA4B;YAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClD,uBAAuB,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjF,uBAAuB,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACxD,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACnF,uBAAuB,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACrE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpF,uBAAuB,CAAC,KAAK,EAAE,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,uBAAuB,CAAC,IAAI,EAAE,CAAC;YACjC,CAAC;YAED,qBAAqB;YACrB,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC1B,gBAAgB,EAAE,CAAC;YACrB,CAAC;YAED,iBAAiB;YACjB,kBAAkB,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,YAAY,EAAE,OAAO,CAAC,MAAM;YAC5B,eAAe,EAAE,OAAO,CAAC,MAAM;YAC/B,kBAAkB;YAClB,uBAAuB;YACvB,gBAAgB;YAChB,oBAAoB,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SACnF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB,CAAC,OAAuB,EAAE,KAAa;QAC9D,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEpF,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YAC1B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAE7C,mCAAmC;YACnC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;gBACxB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,KAAK,IAAI,EAAE,CAAC;gBACd,CAAC;gBACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,qDAAqD;YACrD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACpB,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7C,CAAC;YAED,qBAAqB;YACrB,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC1B,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YAED,2CAA2C;YAC3C,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,cAAc,GAAG,CAAC,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;gBAClG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;oBACnE,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,OAAO;gBACL,GAAG,MAAM;gBACT,cAAc,EAAE,KAAK;aACuB,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/html-cleaner.d.ts

@@ -0,0 +1,62 @@
+export interface CleaningStats {
+    originalSize: number;
+    cleanedSize: number;
+    reductionPercent: number;
+    originalLines: number;
+    cleanedLines: number;
+    elementsRemoved: {
+        scripts: number;
+        styles: number;
+        images: number;
+        ads: number;
+        navigation: number;
+        comments: number;
+    };
+    contentAnalysis: {
+        hasSearchResults: boolean;
+        resultCount: number;
+        hasRichSnippets: boolean;
+        hasAds: boolean;
+        readabilityScore: number;
+    };
+}
+export declare class HtmlCleaner {
+    /**
+     * Extract and clean search results HTML for optimal LLM consumption
+     */
+    static extractSearchResults(html: string): string;
+    /**
+     * Get comprehensive cleaning statistics
+     */
+    static getCleaningStats(originalHtml: string, cleanedHtml: string): CleaningStats;
+    /**
+     * Detect if HTML contains search results
+     */
+    private static detectSearchResults;
+    /**
+     * Count approximate number of search results
+     */
+    private static countSearchResults;
+    /**
+     * Detect rich snippets and structured data
+     */
+    private static detectRichSnippets;
+    /**
+     * Calculate basic readability score
+     */
+    private static calculateReadabilityScore;
+    /**
+     * Extract structured data from HTML (JSON-LD, microdata, etc.)
+     */
+    static extractStructuredData(html: string): any[];
+    /**
+     * Extract meta information from HTML
+     */
+    static extractMetaInfo(html: string): {
+        title?: string;
+        description?: string;
+        keywords?: string;
+        robots?: string;
+        canonical?: string;
+    };
+}