@ignidor/web-search-mcp 1.0.0

package/dist/ranking.js

@@ -0,0 +1,218 @@
+// BM25 and Hybrid Ranking Engine
+import { BM25 } from 'fast-bm25';
+// ============================================================================
+// BM25 Ranker
+// ============================================================================
+export class BM25Ranker {
+    bm25 = null;
+    k1 = 1.5; // Term frequency saturation
+    b = 0.75; // Length normalization
+    constructor(k1, b) {
+        if (k1 !== undefined)
+            this.k1 = k1;
+        if (b !== undefined)
+            this.b = b;
+    }
+    indexDocuments(documents) {
+        // fast-bm25 expects array of objects with string fields
+        const contents = documents.map(d => ({ content: d.content }));
+        this.bm25 = new BM25(contents, { k1: this.k1, b: this.b });
+    }
+    indexSearchResults(results) {
+        const documents = results.map((r, i) => ({
+            id: r.url,
+            content: `${r.title} ${r.description}`,
+            metadata: { ...r, index: i }
+        }));
+        this.indexDocuments(documents);
+    }
+    rank(query, documents) {
+        if (!this.bm25) {
+            this.indexDocuments(documents);
+        }
+        // fast-bm25 returns array of {index, score} sorted by relevance
+        const bm25Results = this.bm25.search(query, documents.length);
+        // Map to our RankedDocument format
+        return bm25Results.map(({ index, score }) => ({
+            doc: documents[index],
+            score
+        }));
+    }
+    rankSearchResults(query, results) {
+        this.indexSearchResults(results);
+        const documents = results.map((r, i) => ({
+            id: r.url,
+            content: `${r.title} ${r.description}`,
+            metadata: { ...r, index: i }
+        }));
+        const ranked = this.rank(query, documents);
+        return ranked.map((r, i) => ({
+            position: i + 1,
+            score: r.score,
+            title: r.doc.metadata.title,
+            url: r.doc.id,
+            snippet: r.doc.metadata.description,
+            provider: r.doc.metadata.provider,
+            domain: this.extractDomain(r.doc.id)
+        }));
+    }
+    extractDomain(url) {
+        try {
+            return new URL(url).hostname;
+        }
+        catch {
+            return 'unknown';
+        }
+    }
+}
+// ============================================================================
+// Hybrid Ranker (BM25 + Freshness + Domain Authority)
+// ============================================================================
+export class HybridRanker {
+    bm25Ranker;
+    config;
+    constructor(config) {
+        this.bm25Ranker = new BM25Ranker();
+        this.config = {
+            alpha: 0.6, // BM25 weight
+            beta: 0.2, // Freshness weight
+            gamma: 0.2, // Domain authority weight
+            ...config
+        };
+    }
+    rank(query, documents) {
+        // First get BM25 scores
+        this.bm25Ranker.indexDocuments(documents);
+        const bm25Ranked = this.bm25Ranker.rank(query, documents);
+        // Then add hybrid scoring
+        return bm25Ranked.map(({ doc, score: bm25Score }) => {
+            const freshnessScore = this.calculateFreshnessScore(doc.metadata?.publishedDate);
+            const domainScore = this.calculateDomainScore(doc.id);
+            const totalScore = this.config.alpha * bm25Score +
+                this.config.beta * freshnessScore +
+                this.config.gamma * domainScore;
+            return {
+                doc,
+                score: totalScore,
+                breakdown: { bm25: bm25Score, freshness: freshnessScore, domain: domainScore }
+            };
+        }).sort((a, b) => b.score - a.score);
+    }
+    rankSearchResults(query, results) {
+        const documents = results.map((r, i) => ({
+            id: r.url,
+            content: `${r.title} ${r.description}`,
+            metadata: { ...r, index: i }
+        }));
+        const ranked = this.rank(query, documents);
+        return ranked.map((r, i) => ({
+            position: i + 1,
+            score: r.score,
+            title: r.doc.metadata.title,
+            url: r.doc.id,
+            snippet: r.doc.metadata.description,
+            provider: r.doc.metadata.provider,
+            domain: this.extractDomain(r.doc.id),
+            publishedDate: r.doc.metadata?.publishedDate
+        }));
+    }
+    calculateFreshnessScore(publishedDate) {
+        if (!publishedDate)
+            return 0.5; // Neutral for unknown dates
+        try {
+            const age = Date.now() - new Date(publishedDate).getTime();
+            const daysOld = age / (1000 * 60 * 60 * 24);
+            // Exponential decay: e^(-lambda * days)
+            // Lambda tuned so 7-day-old content has ~0.5 score
+            const lambda = 0.1;
+            return Math.exp(-lambda * daysOld);
+        }
+        catch {
+            return 0.5;
+        }
+    }
+    calculateDomainScore(url) {
+        try {
+            const domain = new URL(url).hostname.toLowerCase();
+            // High authority domains
+            const highAuthorityDomains = [
+                'wikipedia.org',
+                'github.com',
+                'stackoverflow.com',
+                'reddit.com',
+                'medium.com',
+                'nytimes.com',
+                'bbc.com',
+                'theguardian.com',
+                'developer.mozilla.org',
+                'docs.rs'
+            ];
+            // Academic/government
+            if (domain.endsWith('.edu') || domain.endsWith('.gov')) {
+                return 0.9;
+            }
+            // High authority
+            if (highAuthorityDomains.some(d => domain.includes(d))) {
+                return 1.0;
+            }
+            // Common TLDs
+            if (domain.endsWith('.com')) {
+                return 0.7;
+            }
+            if (domain.endsWith('.org')) {
+                return 0.65;
+            }
+            if (domain.endsWith('.net')) {
+                return 0.6;
+            }
+            // Country-specific TLDs
+            if (domain.endsWith('.io') || domain.endsWith('.dev') || domain.endsWith('.ai')) {
+                return 0.55;
+            }
+            return 0.5; // Default for unknown
+        }
+        catch {
+            return 0.5;
+        }
+    }
+    extractDomain(url) {
+        try {
+            return new URL(url).hostname;
+        }
+        catch {
+            return 'unknown';
+        }
+    }
+}
+// ============================================================================
+// Re-ranking with Full Content (After Crawl4AI Extraction)
+// ============================================================================
+export function rerankWithFullContent(query, results, extractedContent) {
+    const ranker = new BM25Ranker();
+    // Index full content instead of just snippets
+    const documents = results
+        .filter(r => extractedContent.has(r.url))
+        .map(r => ({
+        id: r.url,
+        content: extractedContent.get(r.url),
+        metadata: r
+    }));
+    if (documents.length === 0)
+        return results;
+    ranker.indexDocuments(documents);
+    const ranked = ranker.rank(query, documents);
+    // Merge back with non-extracted results (at the end, with lower scores)
+    const extractedUrls = new Set(extractedContent.keys());
+    const nonExtracted = results
+        .filter(r => !extractedUrls.has(r.url))
+        .map((r, i) => ({
+        doc: { id: r.url, content: `${r.title} ${r.snippet}`, metadata: r },
+        score: 0 // Give non-extracted content lowest score
+    }));
+    const allRanked = [...ranked, ...nonExtracted]
+        .sort((a, b) => b.score - a.score)
+        .map((r, i) => r.doc.metadata)
+        .map((r, i) => ({ ...r, position: i + 1 }));
+    return allRanked;
+}
+//# sourceMappingURL=ranking.js.map

package/dist/ranking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking.js","sourceRoot":"","sources":["../src/ranking.ts"],"names":[],"mappings":"AAAA,iCAAiC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA8CjC,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAE/E,MAAM,OAAO,UAAU;IACb,IAAI,GAAgB,IAAI,CAAC;IACzB,EAAE,GAAW,GAAG,CAAC,CAAE,4BAA4B;IAC/C,CAAC,GAAW,IAAI,CAAC,CAAG,uBAAuB;IAEnD,YAAY,EAAW,EAAE,CAAU;QACjC,IAAI,EAAE,KAAK,SAAS;YAAE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,SAAS;YAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,cAAc,CAAC,SAAqB;QAClC,wDAAwD;QACxD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,kBAAkB,CAAC,OAA0B;QAC3C,MAAM,SAAS,GAAe,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC,CAAC,GAAG;YACT,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE;YACtC,QAAQ,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;SAC7B,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC,KAAa,EAAE,SAAqB;QACvC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;QAED,gEAAgE;QAChE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAK,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QAE/D,mCAAmC;QACnC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YAC5C,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC;YACrB,KAAK;SACN,CAAC,CAAC,CAAC;IACN,CAAC;IAED,iBAAiB,CAAC,KAAa,EAAE,OAA0B;QACzD,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEjC,MAAM,SAAS,GAAe,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC,CAAC,GAAG;YACT,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE;YACtC,QAAQ,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,KAAK;YAC3D,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE;YACb,OAAO,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,WAAW;YACnE,QAAQ,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,QAAQ;YACjE,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;SACrC,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,sDAAsD;AACtD,+EAA+E;AAE/E,MAAM,OAAO,YAAY;IACf,UAAU,CAAa;IACvB,MAAM,CAAoB;IAElC,YAAY,MAAmC;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG;YACZ,KAAK,EAAE,GAAG,EAAI,cAAc;YAC5B,IAAI,EAAE,GAAG,EAAK,mBAAmB;YACjC,KAAK,EAAE,GAAG,EAAI,0BAA0B;YACxC,GAAG,MAAM;SACV,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,KAAa,EAAE,SAAqB;QACvC,wBAAwB;QACxB,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE1D,0BAA0B;QAC1B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,EAAE,aAAuB,CAAC,CAAC;YAC3F,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAY,CAAC,CAAC;YAEhE,MAAM,UAAU,GACd,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,SAAS;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,cAAc;gBACjC,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,WAAW,CAAC;YAElC,OAAO;gBACL,GAAG;gBACH,KAAK,EAAE,UAAU;gBACjB,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE;aAC/E,CAAC;QACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,iBAAiB,CAAC,KAAa,EAAE,OAA0B;QACzD,MAAM,SAAS,GAAe,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC,CAAC,GAAG;YACT,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE;YACtC,QAAQ,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,KAAK;YAC3D,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE;YACb,OAAO,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,WAAW;YACnE,QAAQ,EAAG,CAAC,CAAC,GAAG,CAAC,QAAuC,CAAC,QAAQ;YACjE,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,aAAa,EAAG,CAAC,CAAC,GAAG,CAAC,QAAoC,EAAE,aAAuB;SACpF,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,uBAAuB,CAAC,aAAsB;QACpD,IAAI,CAAC,aAAa;YAAE,OAAO,GAAG,CAAC,CAAC,4BAA4B;QAE5D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YAE5C,wCAAwC;YACxC,mDAAmD;YACnD,MAAM,MAAM,GAAG,GAAG,CAAC;YACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,GAAW;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAEnD,yBAAyB;YACzB,MAAM,oBAAoB,GAAG;gBAC3B,eAAe;gBACf,YAAY;gBACZ,mBAAmB;gBACnB,YAAY;gBACZ,YAAY;gBACZ,aAAa;gBACb,SAAS;gBACT,iBAAiB;gBACjB,uBAAuB;gBACvB,SAAS;aACV,CAAC;YAEF,sBAAsB;YACtB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvD,OAAO,GAAG,CAAC;YACb,CAAC;YAED,iBAAiB;YACjB,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,OAAO,GAAG,CAAC;YACb,CAAC;YAED,cAAc;YACd,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC;YACb,CAAC;YACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,OAAO,GAAG,CAAC;YACb,CAAC;YAED,wBAAwB;YACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChF,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,GAAG,CAAC,CAAC,sBAAsB;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,MAAM,UAAU,qBAAqB,CACnC,KAAa,EACb,OAAuB,EACvB,gBAAqC;IAErC,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAEhC,8CAA8C;IAC9C,MAAM,SAAS,GAAe,OAAO;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACT,EAAE,EAAE,CAAC,CAAC,GAAG;QACT,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAE;QACrC,QAAQ,EAAE,CAAuC;KAClD,CAAC,CAAC,CAAC;IAEN,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAE3C,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAE7C,wEAAwE;IACxE,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,OAAO;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACd,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE;QACnE,KAAK,EAAE,CAAC,CAAC,0CAA0C;KACpD,CAAC,CAAC,CAAC;IAEN,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC;SAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,QAAwB,CAAC;SAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9C,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/search.d.ts

@@ -0,0 +1,15 @@
+export interface RawSearchResult {
+    title: string;
+    url: string;
+    description: string;
+    provider: 'google' | 'duckduckgo';
+}
+export interface ProviderResponse {
+    results: RawSearchResult[];
+    provider: 'google' | 'duckduckgo';
+    hasResults: boolean;
+}
+export declare function performDuckDuckGoSearch(query: string, limit?: number): Promise<ProviderResponse>;
+export declare function performGoogleSearch(query: string, limit?: number): Promise<ProviderResponse>;
+export declare function performSearch(query: string, limit?: number, preferProvider?: 'duckduckgo' | 'google' | 'auto'): Promise<ProviderResponse>;
+//# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AA0CA,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,GAAG,YAAY,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,QAAQ,GAAG,YAAY,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;CACrB;AAsCD,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,GACjB,OAAO,CAAC,gBAAgB,CAAC,CAqD3B;AAMD,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,GACjB,OAAO,CAAC,gBAAgB,CAAC,CA8C3B;AAMD,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,EAClB,cAAc,GAAE,YAAY,GAAG,QAAQ,GAAG,MAAe,GACxD,OAAO,CAAC,gBAAgB,CAAC,CA0B3B"}

package/dist/search.js

@@ -0,0 +1,187 @@
+// Search Module - DuckDuckGo and Google Discovery
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+// ============================================================================
+// Validators (inlined to avoid module resolution issues)
+// ============================================================================
+function isValidUrl(url) {
+    if (!url || typeof url !== 'string')
+        return false;
+    try {
+        const parsed = new URL(url);
+        const hostname = parsed.hostname.toLowerCase();
+        // Block localhost variants
+        if (hostname === 'localhost' ||
+            hostname === '127.0.0.1' ||
+            hostname === '0.0.0.0' ||
+            hostname === '[::1]' ||
+            hostname.startsWith('127.')) {
+            return false;
+        }
+        // Block private IP ranges
+        if (hostname.startsWith('192.168.') ||
+            hostname.startsWith('10.') ||
+            hostname.startsWith('172.16.')) {
+            return false;
+        }
+        // Only allow http/https
+        return url.startsWith('http://') || url.startsWith('https://');
+    }
+    catch {
+        return false;
+    }
+}
+// ============================================================================
+// Configuration
+// ============================================================================
+const USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15'
+];
+const SEARCH_TIMEOUT = 10000; // 10 seconds
+// ============================================================================
+// Helper Functions
+// ============================================================================
+function getRandomUserAgent() {
+    return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+function getCommonHeaders() {
+    return {
+        'User-Agent': getRandomUserAgent(),
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'DNT': '1',
+        'Connection': 'keep-alive'
+    };
+}
+// ============================================================================
+// DuckDuckGo Search
+// ============================================================================
+export async function performDuckDuckGoSearch(query, limit = 10) {
+    try {
+        const response = await axios.get('https://html.duckduckgo.com/html/', {
+            params: { q: query },
+            headers: getCommonHeaders(),
+            timeout: SEARCH_TIMEOUT
+        });
+        const $ = cheerio.load(response.data);
+        const results = [];
+        $('.web-result').each((i, element) => {
+            if (i >= limit)
+                return false;
+            const titleElement = $(element).find('.result__title a.result__a');
+            const snippetElement = $(element).find('.result__snippet');
+            if (titleElement.length) {
+                let url = titleElement.attr('href');
+                const title = titleElement.text().trim();
+                // DuckDuckGo uses redirect URLs, extract the actual URL
+                if (url && url.startsWith('//duckduckgo.com/l/?uddg=')) {
+                    try {
+                        const encodedUrl = url.split('uddg=')[1]?.split('&')[0];
+                        if (encodedUrl) {
+                            url = decodeURIComponent(encodedUrl);
+                        }
+                    }
+                    catch {
+                        return; // Skip if decode fails
+                    }
+                }
+                if (url && isValidUrl(url)) {
+                    results.push({
+                        title,
+                        url,
+                        description: snippetElement.text().trim() || '',
+                        provider: 'duckduckgo'
+                    });
+                }
+            }
+        });
+        return {
+            results,
+            provider: 'duckduckgo',
+            hasResults: results.length > 0
+        };
+    }
+    catch (error) {
+        console.error('[Search] DuckDuckGo failed:', error instanceof Error ? error.message : error);
+        return { results: [], provider: 'duckduckgo', hasResults: false };
+    }
+}
+// ============================================================================
+// Google Search (Fallback)
+// ============================================================================
+export async function performGoogleSearch(query, limit = 10) {
+    try {
+        const response = await axios.get('https://www.google.com/search', {
+            params: {
+                q: query,
+                num: limit
+            },
+            headers: {
+                ...getCommonHeaders(),
+                'Upgrade-Insecure-Requests': '1'
+            },
+            timeout: SEARCH_TIMEOUT
+        });
+        const $ = cheerio.load(response.data);
+        const results = [];
+        $('div.g').each((i, element) => {
+            if (i >= limit)
+                return false;
+            const titleElement = $(element).find('h3');
+            const linkElement = $(element).find('a');
+            const snippetElement = $(element).find('.VwiC3b');
+            if (titleElement.length && linkElement.length) {
+                const url = linkElement.attr('href');
+                if (url && isValidUrl(url)) {
+                    results.push({
+                        title: titleElement.text().trim(),
+                        url,
+                        description: snippetElement.text().trim() || '',
+                        provider: 'google'
+                    });
+                }
+            }
+        });
+        return {
+            results,
+            provider: 'google',
+            hasResults: results.length > 0
+        };
+    }
+    catch (error) {
+        console.error('[Search] Google failed:', error instanceof Error ? error.message : error);
+        return { results: [], provider: 'google', hasResults: false };
+    }
+}
+// ============================================================================
+// Unified Search with Fallback
+// ============================================================================
+export async function performSearch(query, limit = 10, preferProvider = 'auto') {
+    // Validate inputs
+    if (!query || query.trim().length === 0) {
+        return { results: [], provider: 'duckduckgo', hasResults: false };
+    }
+    const actualLimit = Math.min(Math.max(limit, 1), 50);
+    // Try preferred provider first, then fallback
+    let response;
+    if (preferProvider === 'google') {
+        response = await performGoogleSearch(query, actualLimit);
+        if (!response.hasResults) {
+            console.error('[Search] Google failed, trying DuckDuckGo...');
+            response = await performDuckDuckGoSearch(query, actualLimit);
+        }
+    }
+    else {
+        response = await performDuckDuckGoSearch(query, actualLimit);
+        if (!response.hasResults) {
+            console.error('[Search] DuckDuckGo failed, trying Google...');
+            response = await performGoogleSearch(query, actualLimit);
+        }
+    }
+    return response;
+}
+//# sourceMappingURL=search.js.map

package/dist/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,+EAA+E;AAC/E,yDAAyD;AACzD,+EAA+E;AAE/E,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,IAAI,QAAQ,KAAK,WAAW;YACxB,QAAQ,KAAK,WAAW;YACxB,QAAQ,KAAK,SAAS;YACtB,QAAQ,KAAK,OAAO;YACpB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC;YAC1B,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,wBAAwB;QACxB,OAAO,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAmBD,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,WAAW,GAAG;IAClB,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uHAAuH;CACxH,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,aAAa;AAE3C,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO;QACL,YAAY,EAAE,kBAAkB,EAAE;QAClC,QAAQ,EAAE,iEAAiE;QAC3E,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;KAC3B,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,KAAa,EACb,QAAgB,EAAE;IAElB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mCAAmC,EAAE;YACpE,MAAM,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE;YACpB,OAAO,EAAE,gBAAgB,EAAE;YAC3B,OAAO,EAAE,cAAc;SACxB,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC;YAE7B,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YACnE,MAAM,cAAc,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAE3D,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;gBACxB,IAAI,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAEzC,wDAAwD;gBACxD,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,EAAE,CAAC;oBACvD,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;wBACxD,IAAI,UAAU,EAAE,CAAC;4BACf,GAAG,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;wBACvC,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,CAAC,uBAAuB;oBACjC,CAAC;gBACH,CAAC;gBAED,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK;wBACL,GAAG;wBACH,WAAW,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE;wBAC/C,QAAQ,EAAE,YAAY;qBACvB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO;YACP,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;SAC/B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7F,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IACpE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAa,EACb,QAAgB,EAAE;IAElB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,+BAA+B,EAAE;YAChE,MAAM,EAAE;gBACN,CAAC,EAAE,KAAK;gBACR,GAAG,EAAE,KAAK;aACX;YACD,OAAO,EAAE;gBACP,GAAG,gBAAgB,EAAE;gBACrB,2BAA2B,EAAE,GAAG;aACjC;YACD,OAAO,EAAE,cAAc;SACxB,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,CAAC,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC;YAE7B,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,cAAc,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,YAAY,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC9C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrC,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;wBACjC,GAAG;wBACH,WAAW,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE;wBAC/C,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO;YACP,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;SAC/B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACzF,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAChE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+BAA+B;AAC/B,+EAA+E;AAE/E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,QAAgB,EAAE,EAClB,iBAAmD,MAAM;IAEzD,kBAAkB;IAClB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAErD,8CAA8C;IAC9C,IAAI,QAA0B,CAAC;IAE/B,IAAI,cAAc,KAAK,QAAQ,EAAE,CAAC;QAChC,QAAQ,GAAG,MAAM,mBAAmB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC9D,QAAQ,GAAG,MAAM,uBAAuB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,MAAM,uBAAuB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC9D,QAAQ,GAAG,MAAM,mBAAmB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/types/index.d.ts

@@ -0,0 +1,131 @@
+export interface SearchArgs {
+    query: string;
+    limit?: number;
+    rankingMode?: 'bm25' | 'tfidf' | 'hybrid';
+    freshness?: 'day' | 'week' | 'month';
+}
+export interface SearchResult {
+    position: number;
+    score: number;
+    title: string;
+    url: string;
+    snippet: string;
+    provider: 'duckduckgo' | 'google';
+    domain: string;
+    publishedDate?: string;
+}
+export interface CrawlArgs {
+    urls: string[];
+    extractDepth?: 'basic' | 'advanced';
+    includeImages?: boolean;
+    waitForSelector?: string;
+    executeJavascript?: boolean;
+    timeout?: number;
+}
+export interface CrawlResult {
+    url: string;
+    success: boolean;
+    markdown?: string;
+    title?: string;
+    wordCount?: number;
+    images?: string[];
+    links?: number;
+    extractedAt?: string;
+    durationMs?: number;
+    error?: string;
+}
+export interface SearchAndCrawlArgs extends SearchArgs {
+    extractTopN?: number;
+    rerankAfterExtract?: boolean;
+    extractDepth?: 'basic' | 'advanced';
+    includeImages?: boolean;
+}
+export interface SearchAndCrawlResult {
+    searchResults: SearchResult[];
+    extractedContent: CrawlResult[];
+    summary: {
+        totalFetched: number;
+        successful: number;
+        failed: number;
+        totalTimeMs: number;
+    };
+}
+export interface BulkCrawlArgs {
+    urls: string[];
+    concurrency?: number;
+    extractDepth?: 'basic' | 'advanced';
+    includeImages?: boolean;
+    timeout?: number;
+    continueOnError?: boolean;
+}
+export interface BulkCrawlResult {
+    results: CrawlResult[];
+    summary: {
+        total: number;
+        successful: number;
+        failed: number;
+        totalTimeMs: number;
+    };
+}
+export interface SearchWithFiltersArgs extends SearchArgs {
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    site?: string;
+    fileType?: 'pdf' | 'doc' | 'xls' | 'ppt';
+    resultType?: 'web' | 'news' | 'videos' | 'images';
+    language?: string;
+    region?: string;
+}
+export interface Document {
+    id: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+export interface RankedDocument {
+    doc: Document;
+    score: number;
+    breakdown?: {
+        bm25: number;
+        freshness: number;
+        domain: number;
+    };
+}
+export interface HybridScoreConfig {
+    alpha: number;
+    beta: number;
+    gamma: number;
+}
+export interface Crawl4AIConfig {
+    extractDepth?: 'basic' | 'advanced';
+    includeImages?: boolean;
+    waitForSelector?: string;
+    executeJavascript?: boolean;
+    timeout?: number;
+}
+export interface Crawl4AIResponse {
+    success: boolean;
+    url?: string;
+    markdown?: string;
+    cleaned_html?: string;
+    title?: string;
+    links?: {
+        internal: number;
+        external: number;
+    };
+    media?: {
+        images: string[];
+    };
+    error?: string;
+}
+export interface RawSearchResult {
+    title: string;
+    url: string;
+    description: string;
+    provider: 'google' | 'duckduckgo';
+}
+export interface ProviderResponse {
+    results: RawSearchResult[];
+    provider: 'google' | 'duckduckgo';
+    hasResults: boolean;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC1C,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,YAAY,GAAG,QAAQ,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAMD,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD,MAAM,WAAW,kBAAmB,SAAQ,UAAU;IACpD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,gBAAgB,EAAE,WAAW,EAAE,CAAC;IAChC,OAAO,EAAE;QACP,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAMD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAMD,MAAM,WAAW,qBAAsB,SAAQ,UAAU;IACvD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;IACzC,UAAU,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,QAAQ,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAMD,MAAM,WAAW,cAAc;IAC7B,YAAY,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,KAAK,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,GAAG,YAAY,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,QAAQ,GAAG,YAAY,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;CACrB"}

package/dist/types/index.js

@@ -0,0 +1,3 @@
+// Type definitions for Crawl4AI Web-Search MCP
+export {};
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,+CAA+C"}

package/dist/utils/concurrency.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Creates a concurrency limiter for controlling parallel operations.
+ * @param concurrency - Maximum number of concurrent operations (default: 5)
+ * @returns A limit function that wraps async operations
+ */
+export declare function createConcurrencyLimiter(concurrency?: number): import("p-limit").LimitFunction;
+/**
+ * Processes items in batches with concurrency control.
+ * @param items - Array of items to process
+ * @param processor - Async function to process each item
+ * @param concurrency - Maximum concurrent operations
+ * @returns Array of results in the same order as input
+ */
+export declare function processBatch<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, concurrency?: number): Promise<R[]>;
+/**
+ * Processes items with graceful error handling.
+ * Failed items return undefined instead of rejecting the entire batch.
+ * @param items - Array of items to process
+ * @param processor - Async function to process each item
+ * @param concurrency - Maximum concurrent operations
+ * @returns Array of successful results or undefined for failed items
+ */
+export declare function processBatchWithErrorHandling<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, concurrency?: number): Promise<Array<R | undefined>>;
+//# sourceMappingURL=concurrency.d.ts.map

package/dist/utils/concurrency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concurrency.d.ts","sourceRoot":"","sources":["../../src/utils/concurrency.ts"],"names":[],"mappings":"AAQA;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,WAAW,GAAE,MAAU,mCAE/D;AAMD;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,CAAC,EACrC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,WAAW,GAAE,MAAU,GACtB,OAAO,CAAC,CAAC,EAAE,CAAC,CAQd;AAMD;;;;;;;GAOG;AACH,wBAAsB,6BAA6B,CAAC,CAAC,EAAE,CAAC,EACtD,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,WAAW,GAAE,MAAU,GACtB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAe/B"}