@framers/agentos-ext-web-search 1.1.0 → 1.1.1

package/src/services/searchProvider.ts

@@ -5,6 +5,7 @@ export interface SearchProviderConfig {
   serperApiKey?: string;
   serpApiKey?: string;
   braveApiKey?: string;
+  searxngUrl?: string;
   maxRetries?: number;
   rateLimit?: {
     maxRequests: number;
@@ -36,6 +37,37 @@ export interface ProviderResponse {
   };
 }
 
+/**
+ * A search result enriched with cross-provider agreement data.
+ */
+export interface MultiSearchResult extends SearchResult {
+  /** Which providers returned this URL */
+  providers: string[];
+  /** Number of providers that returned this URL */
+  agreementCount: number;
+  /** Confidence score (0-100) based on cross-provider agreement + position */
+  confidenceScore: number;
+  /** The position this result appeared at in each provider's results */
+  providerPositions: Record<string, number>;
+}
+
+/**
+ * Response from a multi-provider parallel search aggregation.
+ */
+export interface MultiSearchResponse {
+  results: MultiSearchResult[];
+  metadata: {
+    query: string;
+    timestamp: string;
+    totalResponseTime: number;
+    providersQueried: string[];
+    providersSucceeded: string[];
+    providersFailed: string[];
+    totalRawResults: number;
+    deduplicatedCount: number;
+  };
+}
+
 /**
  * Service for managing multiple search providers with fallback support
  * 
@@ -62,12 +94,15 @@ export class SearchProviderService {
    */
   constructor(config: SearchProviderConfig) {
     this.config = {
-      maxRetries: 3,
-      rateLimit: {
+      serperApiKey: config.serperApiKey,
+      serpApiKey: config.serpApiKey,
+      braveApiKey: config.braveApiKey,
+      searxngUrl: config.searxngUrl,
+      maxRetries: config.maxRetries ?? 3,
+      rateLimit: config.rateLimit ?? {
         maxRequests: 10,
-        windowMs: 60000
+        windowMs: 60000,
       },
-      ...config
     };
     this.rateLimitState = new Map();
   }
@@ -84,10 +119,11 @@ export class SearchProviderService {
    * @throws {Error} If all providers fail and no fallback is available
    */
   async search(
-    query: string, 
-    options: { 
-      maxResults?: number; 
-      provider?: string 
+    query: string,
+    options: {
+      maxResults?: number;
+      provider?: string;
+      category?: string;
     } = {}
   ): Promise<ProviderResponse> {
     const startTime = Date.now();
@@ -95,23 +131,23 @@ export class SearchProviderService {
     
     // If specific provider requested, try only that one
     if (options.provider) {
-      return this.searchWithProvider(query, options.provider, maxResults, startTime);
+      return this.searchWithProvider(query, options.provider, maxResults, startTime, options.category);
     }
-    
+
     // Try providers in order of preference
     const providers = this.getAvailableProviders();
-    
+
     for (const provider of providers) {
       try {
         if (await this.checkRateLimit(provider)) {
-          return await this.searchWithProvider(query, provider, maxResults, startTime);
+          return await this.searchWithProvider(query, provider, maxResults, startTime, options.category);
         }
       } catch (error) {
         console.warn(`Provider ${provider} failed:`, error);
         continue;
       }
     }
-    
+
     // Final fallback to DuckDuckGo (no API key required)
     return this.searchDuckDuckGo(query, maxResults, startTime);
   }
@@ -130,10 +166,11 @@ export class SearchProviderService {
     query: string,
     provider: string,
     maxResults: number,
-    startTime: number
+    startTime: number,
+    category?: string
   ): Promise<ProviderResponse> {
     let results: SearchResult[];
-    
+
     switch (provider) {
       case 'serper':
         results = await this.searchSerper(query, maxResults);
@@ -144,6 +181,9 @@ export class SearchProviderService {
       case 'brave':
         results = await this.searchBrave(query, maxResults);
         break;
+      case 'searxng':
+        results = await this.searchSearXNG(query, maxResults, { categories: category });
+        break;
       case 'duckduckgo':
       default:
         results = await this.searchDuckDuckGoAPI(query, maxResults);
@@ -163,17 +203,17 @@ export class SearchProviderService {
   
   /**
    * Gets list of available providers based on configured API keys
-   * 
-   * @private
+   *
    * @returns {string[]} Array of available provider names
    */
-  private getAvailableProviders(): string[] {
+  public getAvailableProviders(): string[] {
     const providers: string[] = [];
     
     if (this.config.serperApiKey) providers.push('serper');
     if (this.config.serpApiKey) providers.push('serpapi');
     if (this.config.braveApiKey) providers.push('brave');
-    
+    if (this.config.searxngUrl) providers.push('searxng');
+
     return providers;
   }
   
@@ -224,7 +264,7 @@ export class SearchProviderService {
     
     if (!response.ok) throw new Error(`Serper API error: ${response.statusText}`);
     
-    const data = await response.json();
+    const data = (await response.json()) as any;
     return (data.organic || []).map((item: any) => ({
       title: item.title,
       url: item.link,
@@ -251,7 +291,7 @@ export class SearchProviderService {
     const response = await fetch(`https://serpapi.com/search?${params}`);
     if (!response.ok) throw new Error(`SerpAPI error: ${response.statusText}`);
     
-    const data = await response.json();
+    const data = (await response.json()) as any;
     return (data.organic_results || []).map((item: any) => ({
       title: item.title,
       url: item.link,
@@ -282,7 +322,7 @@ export class SearchProviderService {
     
     if (!response.ok) throw new Error(`Brave API error: ${response.statusText}`);
     
-    const data = await response.json();
+    const data = (await response.json()) as any;
     return (data.web?.results || []).map((item: any, index: number) => ({
       title: item.title,
       url: item.url,
@@ -350,7 +390,7 @@ export class SearchProviderService {
     });
     
     const response = await fetch(`https://api.duckduckgo.com/?${params}`);
-    const data = await response.json();
+    const data = (await response.json()) as any;
     
     const results: SearchResult[] = [];
     
@@ -379,9 +419,169 @@ export class SearchProviderService {
     return results.slice(0, maxResults);
   }
   
+  /**
+   * Search using a self-hosted SearXNG metasearch instance.
+   *
+   * @private
+   * @param {string} query - Search query
+   * @param {number} maxResults - Maximum results
+   * @param {Object} [options] - SearXNG-specific options
+   * @param {string} [options.categories] - SearXNG categories (e.g. 'general', 'news', 'images')
+   * @returns {Promise<SearchResult[]>} Search results
+   */
+  private async searchSearXNG(
+    query: string,
+    maxResults: number,
+    options?: { categories?: string }
+  ): Promise<SearchResult[]> {
+    const params = new URLSearchParams({
+      q: query,
+      format: 'json',
+      categories: options?.categories || 'general',
+    });
+
+    const baseUrl = this.config.searxngUrl!.replace(/\/+$/, '');
+    const response = await fetch(`${baseUrl}/search?${params}`);
+
+    if (!response.ok) throw new Error(`SearXNG API error: ${response.statusText}`);
+
+    const data = (await response.json()) as any;
+    return (data.results || []).slice(0, maxResults).map((item: any, index: number) => ({
+      title: item.title || '',
+      url: item.url || '',
+      snippet: item.content || '',
+      position: index + 1,
+    }));
+  }
+
+  /**
+   * Normalizes a URL for deduplication — strips www, tracking params, trailing slash.
+   */
+  static normalizeUrl(rawUrl: string): string {
+    try {
+      const url = new URL(rawUrl);
+      url.hostname = url.hostname.replace(/^www\./, '');
+      const trackingParams = [
+        'utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term',
+        'fbclid', 'gclid', 'ref', 'source', 'sxsrf', 'ei', 'ved',
+      ];
+      for (const param of trackingParams) url.searchParams.delete(param);
+      url.searchParams.sort();
+      url.pathname = url.pathname.replace(/\/+$/, '') || '/';
+      return url.toString();
+    } catch {
+      return rawUrl.toLowerCase().trim();
+    }
+  }
+
+  /**
+   * Searches ALL available providers in parallel, then merges, deduplicates,
+   * and reranks results by cross-provider agreement.
+   */
+  async multiSearch(
+    query: string,
+    options: { maxResults?: number } = {},
+  ): Promise<MultiSearchResponse> {
+    const startTime = Date.now();
+    const maxResults = options.maxResults || 10;
+
+    const providers = [...new Set([...this.getAvailableProviders(), 'duckduckgo'])];
+
+    const providerResults = await Promise.allSettled(
+      providers.map(async (provider) => {
+        try {
+          if (provider !== 'duckduckgo' && !(await this.checkRateLimit(provider))) {
+            throw new Error(`Rate limit exceeded for ${provider}`);
+          }
+          const response = await this.searchWithProvider(query, provider, maxResults, startTime);
+          return { provider, results: response.results, success: true as const };
+        } catch {
+          return { provider, results: [] as SearchResult[], success: false as const };
+        }
+      }),
+    );
+
+    const succeeded: string[] = [];
+    const failed: string[] = [];
+    const allResults: Array<{ provider: string; result: SearchResult; position: number }> = [];
+
+    for (const outcome of providerResults) {
+      if (outcome.status === 'fulfilled') {
+        const { provider, results, success } = outcome.value;
+        if (success && results.length > 0) {
+          succeeded.push(provider);
+          results.forEach((result, index) => {
+            allResults.push({ provider, result, position: index + 1 });
+          });
+        } else {
+          failed.push(provider);
+        }
+      } else {
+        failed.push('unknown');
+      }
+    }
+
+    // Deduplicate and merge by normalized URL
+    const merged = new Map<string, MultiSearchResult>();
+
+    for (const { provider, result, position } of allResults) {
+      const key = SearchProviderService.normalizeUrl(result.url);
+
+      if (merged.has(key)) {
+        const existing = merged.get(key)!;
+        if (!existing.providers.includes(provider)) {
+          existing.providers.push(provider);
+          existing.agreementCount = existing.providers.length;
+        }
+        existing.providerPositions[provider] = position;
+        if ((result.snippet?.length || 0) > (existing.snippet?.length || 0)) {
+          existing.snippet = result.snippet;
+        }
+      } else {
+        merged.set(key, {
+          title: result.title,
+          url: result.url,
+          snippet: result.snippet,
+          position,
+          providers: [provider],
+          agreementCount: 1,
+          confidenceScore: 0,
+          providerPositions: { [provider]: position },
+        });
+      }
+    }
+
+    // Score and rank
+    const totalProviders = succeeded.length || 1;
+    const scoredResults = Array.from(merged.values()).map((r) => {
+      const agreementBonus = (r.agreementCount / totalProviders) * 50;
+      const positions = Object.values(r.providerPositions);
+      const avgNorm = positions.reduce((s, p) => s + (1 - (p - 1) / maxResults), 0) / positions.length;
+      const positionScore = avgNorm * 30;
+      r.confidenceScore = Math.round(Math.min(100, agreementBonus + positionScore + 20));
+      return r;
+    });
+
+    scoredResults.sort((a, b) => b.confidenceScore - a.confidenceScore);
+
+    return {
+      results: scoredResults.slice(0, maxResults),
+      metadata: {
+        query,
+        timestamp: new Date().toISOString(),
+        totalResponseTime: Date.now() - startTime,
+        providersQueried: providers,
+        providersSucceeded: succeeded,
+        providersFailed: failed,
+        totalRawResults: allResults.length,
+        deduplicatedCount: merged.size,
+      },
+    };
+  }
+
   /**
    * Gets recommended search providers with signup information
-   * 
+   *
    * @static
    * @returns {Array} Array of provider recommendations
    */
@@ -405,6 +605,12 @@ export class SearchProviderService {
         freeQuota: '2,000 queries/month',
         description: 'Privacy-focused search API'
       },
+      {
+        name: 'SearXNG',
+        signupUrl: 'https://docs.searxng.org (self-hosted)',
+        freeQuota: 'Unlimited (self-hosted)',
+        description: 'Self-hosted metasearch engine aggregating 244+ search services'
+      },
       {
         name: 'DuckDuckGo',
         signupUrl: 'No signup required',
@@ -413,4 +619,4 @@ export class SearchProviderService {
       }
     ];
   }
-}
+}