gthinking 1.3.0 → 2.1.1

package/search-discovery.ts

@@ -1,700 +0,0 @@
-/**
- * Search & Discovery Module
- * Real-time web search with multi-source aggregation and intelligent filtering
- */
-
-import {
-  SearchQuery,
-  SearchResult,
-  SearchSession,
-  SearchFilters,
-  AggregatedResult,
-  SourceType,
-  ResultMetadata,
-  ConfidenceLevel,
-  ThinkingEvent,
-  ThinkingError,
-  ThinkingStage
-} from './types';
-import { EventEmitter } from 'events';
-import { llmService } from './llm-service';
-
-// ============================================================================
-// SEARCH PROVIDER INTERFACES
-// ============================================================================
-
-interface SearchProvider {
-  name: string;
-  search(query: string, filters: SearchFilters, maxResults: number): Promise<SearchResult[]>;
-  getCredibilityScore(url: string): number;
-}
-
-// ============================================================================
-// WEB SEARCH PROVIDER IMPLEMENTATION
-// ============================================================================
-
-class WebSearchProvider implements SearchProvider {
-  name = 'web_search';
-
-  async search(
-    query: string,
-    filters: SearchFilters,
-    maxResults: number
-  ): Promise<SearchResult[]> {
-    try {
-      const results = await this.searchWithLLM(query, maxResults);
-      if (results.length > 0) {
-        return this.applyFilters(results, filters);
-      }
-    } catch (error) {
-      console.warn('LLM search failed, falling back to simulation', error);
-    }
-
-    // In real implementation, this would call actual search APIs
-    // For demonstration, we simulate search results
-    const simulatedResults: SearchResult[] = [];
-    
-    // Generate diverse simulated results based on query
-    const topics = this.extractTopics(query);
-    
-    for (let i = 0; i < maxResults; i++) {
-      const topic = topics[i % topics.length];
-      const result: SearchResult = {
-        id: `web_${Date.now()}_${i}`,
-        title: this.generateTitle(topic, i),
-        url: `https://example.com/article/${topic.replace(/\s+/g, '-')}-${i}`,
-        snippet: this.generateSnippet(topic, query),
-        source: SourceType.WEB,
-        credibility: this.calculateCredibility(i),
-        relevance: this.calculateRelevance(query, topic, i),
-        timestamp: new Date(),
-        metadata: this.generateMetadata(topic)
-      };
-      simulatedResults.push(result);
-    }
-
-    return this.applyFilters(simulatedResults, filters);
-  }
-
-  private async searchWithLLM(query: string, maxResults: number): Promise<SearchResult[]> {
-    const prompt = `
-      Act as a search engine. Query: "${query}"
-      Generate ${maxResults} relevant search results based on your knowledge.
-      
-      Return strictly JSON with an array of objects:
-      [
-        {
-          "title": "string",
-          "url": "https://... (invent a plausible URL)",
-          "snippet": "string (1-2 sentences summarizing the content)",
-          "credibility": number (0-1)
-        }
-      ]
-    `;
-
-    const response = await llmService.generateText({ prompt });
-    const jsonMatch = response.match(/\[[\s\S]*\]/); // Match array
-    
-    if (jsonMatch) {
-      const data = JSON.parse(jsonMatch[0]);
-      return data.map((item: any, i: number) => ({
-        id: `web_llm_${Date.now()}_${i}`,
-        title: item.title,
-        url: item.url,
-        snippet: item.snippet,
-        source: SourceType.WEB,
-        credibility: item.credibility || 0.7,
-        relevance: 0.9, // Assumed relevant
-        timestamp: new Date(),
-        metadata: { wordCount: 1000, citations: 10 }
-      }));
-    }
-    return [];
-  }
-
-  getCredibilityScore(url: string): number {
-    const credibleDomains = [
-      'edu', 'gov', 'ac.uk', 'ac.jp', 'arxiv.org',
-      'nature.com', 'science.org', 'ieee.org', 'acm.org'
-    ];
-    
-    const lowCredibilityDomains = [
-      'blogspot', 'wordpress', 'medium', 'wix', 'weebly'
-    ];
-
-    if (credibleDomains.some(d => url.includes(d))) return 0.9;
-    if (lowCredibilityDomains.some(d => url.includes(d))) return 0.4;
-    return 0.6;
-  }
-
-  private extractTopics(query: string): string[] {
-    // Simple topic extraction - in real implementation, use NLP
-    const commonWords = ['what', 'how', 'why', 'when', 'where', 'who', 'is', 'are', 'the', 'a', 'an'];
-    return query
-      .toLowerCase()
-      .split(/\s+/)
-      .filter(word => !commonWords.includes(word) && word.length > 3);
-  }
-
-  private generateTitle(topic: string, index: number): string {
-    const templates = [
-      `Complete Guide to ${topic}`,
-      `${topic}: Everything You Need to Know`,
-      `Understanding ${topic} in Depth`,
-      `The Future of ${topic}`,
-      `${topic} Best Practices and Strategies`,
-      `Advanced ${topic} Techniques`,
-      `${topic} Case Studies and Examples`,
-      `Breaking Down ${topic}: A Comprehensive Analysis`
-    ];
-    return templates[index % templates.length];
-  }
-
-  private generateSnippet(topic: string, query: string): string {
-    return `This comprehensive resource about ${topic} provides detailed insights related to "${query}". ` +
-           `It covers key concepts, practical applications, and expert perspectives on the subject matter.`;
-  }
-
-  private calculateCredibility(index: number): number {
-    // Higher ranked results typically have higher credibility
-    return Math.max(0.3, 0.95 - (index * 0.05));
-  }
-
-  private calculateRelevance(query: string, topic: string, index: number): number {
-    const queryWords = query.toLowerCase().split(/\s+/);
-    const topicWords = topic.toLowerCase().split(/\s+/);
-    const matchCount = topicWords.filter(w => queryWords.includes(w)).length;
-    const baseRelevance = matchCount / Math.max(queryWords.length, topicWords.length);
-    return Math.min(0.95, baseRelevance + (0.9 - index * 0.1));
-  }
-
-  private generateMetadata(topic: string): ResultMetadata {
-    return {
-      author: `Expert on ${topic}`,
-      publishDate: new Date(Date.now() - Math.random() * 365 * 24 * 60 * 60 * 1000),
-      wordCount: Math.floor(1000 + Math.random() * 4000),
-      readingTime: Math.floor(5 + Math.random() * 20),
-      relatedTopics: [topic, 'related-concept-1', 'related-concept-2'],
-      citations: Math.floor(Math.random() * 50)
-    };
-  }
-
-  private applyFilters(results: SearchResult[], filters: SearchFilters): SearchResult[] {
-    return results.filter(result => {
-      if (filters.minCredibility && result.credibility < filters.minCredibility) {
-        return false;
-      }
-      if (filters.domains && filters.domains.length > 0) {
-        const domainMatch = filters.domains.some(d => result.url.includes(d));
-        if (!domainMatch) return false;
-      }
-      if (filters.excludeDomains) {
-        const excluded = filters.excludeDomains.some(d => result.url.includes(d));
-        if (excluded) return false;
-      }
-      return true;
-    });
-  }
-}
-
-// ============================================================================
-// KNOWLEDGE BASE SEARCH PROVIDER
-// ============================================================================
-
-class KnowledgeBaseProvider implements SearchProvider {
-  name = 'knowledge_base';
-  private knowledgeStore: Map<string, unknown> = new Map();
-
-  async search(
-    query: string,
-    filters: SearchFilters,
-    maxResults: number
-  ): Promise<SearchResult[]> {
-    const results: SearchResult[] = [];
-    const queryLower = query.toLowerCase();
-
-    // Search through stored knowledge
-    for (const [key, value] of this.knowledgeStore.entries()) {
-      if (key.toLowerCase().includes(queryLower)) {
-        results.push({
-          id: `kb_${key}`,
-          title: key,
-          url: `knowledge://${key}`,
-          snippet: JSON.stringify(value).substring(0, 200),
-          source: SourceType.KNOWLEDGE_BASE,
-          credibility: 0.95,
-          relevance: 0.9,
-          timestamp: new Date(),
-          metadata: { wordCount: JSON.stringify(value).length }
-        });
-      }
-      if (results.length >= maxResults) break;
-    }
-
-    return results;
-  }
-
-  getCredibilityScore(): number {
-    return 0.95; // Knowledge base has high credibility
-  }
-
-  addKnowledge(key: string, value: unknown): void {
-    this.knowledgeStore.set(key, value);
-  }
-
-  getKnowledge(key: string): unknown {
-    return this.knowledgeStore.get(key);
-  }
-}
-
-// ============================================================================
-// CACHE SEARCH PROVIDER
-// ============================================================================
-
-class CacheProvider implements SearchProvider {
-  name = 'cache';
-  private cache: Map<string, { result: SearchResult; timestamp: number; ttl: number }> = new Map();
-  private defaultTTL = 3600000; // 1 hour
-
-  async search(
-    query: string,
-    _filters: SearchFilters,
-    maxResults: number
-  ): Promise<SearchResult[]> {
-    const results: SearchResult[] = [];
-    const now = Date.now();
-
-    for (const [key, cached] of this.cache.entries()) {
-      if (key.includes(query) && now - cached.timestamp < cached.ttl) {
-        results.push(cached.result);
-      }
-      if (results.length >= maxResults) break;
-    }
-
-    return results;
-  }
-
-  getCredibilityScore(): number {
-    return 0.8;
-  }
-
-  cacheResult(query: string, result: SearchResult, ttl?: number): void {
-    this.cache.set(query, {
-      result,
-      timestamp: Date.now(),
-      ttl: ttl || this.defaultTTL
-    });
-  }
-
-  invalidate(query: string): void {
-    this.cache.delete(query);
-  }
-
-  clear(): void {
-    this.cache.clear();
-  }
-}
-
-// ============================================================================
-// MAIN SEARCH & DISCOVERY ENGINE
-// ============================================================================
-
-export class SearchDiscoveryEngine extends EventEmitter {
-  private providers: Map<SourceType, SearchProvider> = new Map();
-  private sessions: Map<string, SearchSession> = new Map();
-  private knowledgeBase: KnowledgeBaseProvider;
-  private cache: CacheProvider;
-
-  constructor() {
-    super();
-    this.knowledgeBase = new KnowledgeBaseProvider();
-    this.cache = new CacheProvider();
-    this.registerDefaultProviders();
-  }
-
-  private registerDefaultProviders(): void {
-    this.providers.set(SourceType.WEB, new WebSearchProvider());
-    this.providers.set(SourceType.KNOWLEDGE_BASE, this.knowledgeBase);
-    this.providers.set(SourceType.CACHE, this.cache);
-  }
-
-  /**
-   * Execute a search query across multiple sources
-   */
-  async search(
-    query: string,
-    options: {
-      sources?: SourceType[];
-      filters?: SearchFilters;
-      maxResults?: number;
-      timeout?: number;
-    } = {}
-  ): Promise<SearchResult[]> {
-    const {
-      sources = [SourceType.WEB, SourceType.KNOWLEDGE_BASE, SourceType.CACHE],
-      filters = {},
-      maxResults = 10,
-      timeout = 30000
-    } = options;
-
-    const searchId = this.generateId();
-    const startTime = Date.now();
-
-    this.emit('search_start', {
-      id: searchId,
-      type: 'stage_start',
-      stage: ThinkingStage.SEARCH,
-      timestamp: new Date(),
-      data: { query, sources },
-      sessionId: searchId
-    } as ThinkingEvent);
-
-    try {
-      // Check cache first
-      const cachedResults = await this.searchWithProvider(
-        SourceType.CACHE,
-        query,
-        filters,
-        maxResults
-      );
-
-      if (cachedResults.length > 0) {
-        this.emit('search_complete', {
-          id: searchId,
-          type: 'search_result',
-          stage: ThinkingStage.SEARCH,
-          timestamp: new Date(),
-          data: { results: cachedResults, fromCache: true },
-          sessionId: searchId
-        } as ThinkingEvent);
-        return cachedResults;
-      }
-
-      // Execute searches across all specified sources in parallel
-      const searchPromises = sources
-        .filter(s => s !== SourceType.CACHE) // Already checked cache
-        .map(source => this.searchWithProvider(source, query, filters, maxResults));
-
-      const resultsArrays = await Promise.allSettled(searchPromises);
-      
-      let allResults: SearchResult[] = [];
-      resultsArrays.forEach((result, index) => {
-        if (result.status === 'fulfilled') {
-          allResults = allResults.concat(result.value);
-        } else {
-          console.warn(`Search from ${sources[index]} failed:`, result.reason);
-        }
-      });
-
-      // Rank and filter results
-      const rankedResults = this.rankResults(allResults, query).slice(0, maxResults);
-
-      // Cache top results
-      rankedResults.forEach(result => {
-        this.cache.cacheResult(query, result);
-      });
-
-      const processingTime = Date.now() - startTime;
-
-      this.emit('search_complete', {
-        id: searchId,
-        type: 'search_result',
-        stage: ThinkingStage.SEARCH,
-        timestamp: new Date(),
-        data: { 
-          results: rankedResults, 
-          fromCache: false,
-          processingTime,
-          sourcesQueried: sources.length
-        },
-        sessionId: searchId
-      } as ThinkingEvent);
-
-      return rankedResults;
-
-    } catch (error) {
-      this.emit('search_error', {
-        id: searchId,
-        type: 'stage_error',
-        stage: ThinkingStage.SEARCH,
-        timestamp: new Date(),
-        data: { error },
-        sessionId: searchId
-      } as ThinkingEvent);
-
-      throw new ThinkingError(
-        `Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
-        ThinkingStage.SEARCH,
-        true,
-        error instanceof Error ? error : undefined
-      );
-    }
-  }
-
-  /**
-   * Multi-query search for comprehensive coverage
-   */
-  async multiSearch(
-    queries: string[],
-    options: {
-      sources?: SourceType[];
-      filters?: SearchFilters;
-      maxResultsPerQuery?: number;
-      aggregate?: boolean;
-    } = {}
-  ): Promise<SearchResult[] | AggregatedResult[]> {
-    const { aggregate = true, maxResultsPerQuery = 5 } = options;
-
-    const sessionId = this.generateId();
-    const session: SearchSession = {
-      id: sessionId,
-      queries: [],
-      results: [],
-      aggregatedResults: [],
-      startTime: new Date()
-    };
-
-    this.sessions.set(sessionId, session);
-
-    // Execute all queries
-    const searchPromises = queries.map(async (query) => {
-      const results = await this.search(query, {
-        ...options,
-        maxResults: maxResultsPerQuery
-      });
-      return { query, results };
-    });
-
-    const searchResults = await Promise.all(searchPromises);
-
-    // Collect all results
-    searchResults.forEach(({ query, results }) => {
-      session.queries.push({
-        id: this.generateId(),
-        query,
-        sources: options.sources || [SourceType.WEB],
-        filters: options.filters || {},
-        maxResults: maxResultsPerQuery,
-        timeout: 30000,
-        timestamp: new Date()
-      });
-      session.results.push(...results);
-    });
-
-    if (aggregate) {
-      session.aggregatedResults = this.aggregateResults(session.results);
-      return session.aggregatedResults;
-    }
-
-    return session.results;
-  }
-
-  /**
-   * Aggregate results by topic and identify consensus/conflicts
-   */
-  aggregateResults(results: SearchResult[]): AggregatedResult[] {
-    const topicMap = new Map<string, SearchResult[]>();
-
-    // Group by topic (simplified - in real implementation use NLP clustering)
-    results.forEach(result => {
-      const topic = this.extractMainTopic(result.title);
-      if (!topicMap.has(topic)) {
-        topicMap.set(topic, []);
-      }
-      topicMap.get(topic)!.push(result);
-    });
-
-    const aggregated: AggregatedResult[] = [];
-
-    topicMap.forEach((topicResults, topic) => {
-      // Calculate consensus score
-      const avgCredibility = topicResults.reduce((sum, r) => sum + r.credibility, 0) / topicResults.length;
-      const avgRelevance = topicResults.reduce((sum, r) => sum + r.relevance, 0) / topicResults.length;
-      const consensusScore = (avgCredibility + avgRelevance) / 2;
-
-      // Check for conflicting information
-      const conflictingInfo = this.detectConflicts(topicResults);
-
-      // Extract key insights
-      const keyInsights = this.extractKeyInsights(topicResults);
-
-      aggregated.push({
-        topic,
-        results: topicResults,
-        consensusScore,
-        conflictingInfo,
-        keyInsights
-      });
-    });
-
-    // Sort by consensus score
-    return aggregated.sort((a, b) => b.consensusScore - a.consensusScore);
-  }
-
-  /**
-   * Rank results based on multiple factors
-   */
-  private rankResults(results: SearchResult[], query: string): SearchResult[] {
-    return results
-      .map(result => ({
-        ...result,
-        // Calculate composite score
-        compositeScore: this.calculateCompositeScore(result, query)
-      }))
-      .sort((a, b) => (b as any).compositeScore - (a as any).compositeScore);
-  }
-
-  private calculateCompositeScore(result: SearchResult, query: string): number {
-    const relevanceWeight = 0.4;
-    const credibilityWeight = 0.3;
-    const freshnessWeight = 0.15;
-    const engagementWeight = 0.15;
-
-    // Relevance score
-    const relevanceScore = result.relevance;
-
-    // Credibility score
-    const credibilityScore = result.credibility;
-
-    // Freshness score (newer is better)
-    const ageInDays = (Date.now() - result.timestamp.getTime()) / (1000 * 60 * 60 * 24);
-    const freshnessScore = Math.max(0, 1 - ageInDays / 365);
-
-    // Engagement score (based on citations, reading time)
-    const engagementScore = Math.min(1, (result.metadata.citations || 0) / 100);
-
-    return (
-      relevanceScore * relevanceWeight +
-      credibilityScore * credibilityWeight +
-      freshnessScore * freshnessWeight +
-      engagementScore * engagementWeight
-    );
-  }
-
-  /**
-   * Search with a specific provider
-   */
-  private async searchWithProvider(
-    source: SourceType,
-    query: string,
-    filters: SearchFilters,
-    maxResults: number
-  ): Promise<SearchResult[]> {
-    const provider = this.providers.get(source);
-    if (!provider) {
-      throw new Error(`No provider registered for source: ${source}`);
-    }
-    return provider.search(query, filters, maxResults);
-  }
-
-  /**
-   * Add custom search provider
-   */
-  registerProvider(source: SourceType, provider: SearchProvider): void {
-    this.providers.set(source, provider);
-  }
-
-  /**
-   * Add knowledge to knowledge base
-   */
-  addKnowledge(key: string, value: unknown): void {
-    this.knowledgeBase.addKnowledge(key, value);
-  }
-
-  /**
-   * Get search session by ID
-   */
-  getSession(sessionId: string): SearchSession | undefined {
-    return this.sessions.get(sessionId);
-  }
-
-  /**
-   * Clear all sessions
-   */
-  clearSessions(): void {
-    this.sessions.clear();
-  }
-
-  // Helper methods
-  private generateId(): string {
-    return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-  }
-
-  private extractMainTopic(title: string): string {
-    // Simplified topic extraction
-    const words = title.toLowerCase().split(/\s+/);
-    const significantWords = words.filter(w => w.length > 4);
-    return significantWords.slice(0, 3).join(' ') || 'general';
-  }
-
-  private detectConflicts(results: SearchResult[]): boolean {
-    // Simplified conflict detection
-    // In real implementation, use NLP to compare content
-    const snippets = results.map(r => r.snippet.toLowerCase());
-    const contradictoryTerms = ['not', 'never', 'false', 'incorrect', 'myth'];
-    
-    return snippets.some(snippet => 
-      contradictoryTerms.some(term => snippet.includes(term))
-    );
-  }
-
-  private extractKeyInsights(results: SearchResult[]): string[] {
-    // Simplified insight extraction
-    const insights: string[] = [];
-    const seenTopics = new Set<string>();
-
-    results.forEach(result => {
-      const topic = this.extractMainTopic(result.title);
-      if (!seenTopics.has(topic)) {
-        seenTopics.add(topic);
-        insights.push(`${topic}: ${result.snippet.substring(0, 100)}...`);
-      }
-    });
-
-    return insights.slice(0, 5);
-  }
-}
-
-// ============================================================================
-// EXPORT SINGLETON INSTANCE
-// ============================================================================
-
-export const searchEngine = new SearchDiscoveryEngine();
-
-// ============================================================================
-// EXAMPLE USAGE
-// ============================================================================
-
-/*
-// Basic search
-const results = await searchEngine.search('artificial intelligence trends 2024', {
-  sources: [SourceType.WEB, SourceType.KNOWLEDGE_BASE],
-  maxResults: 10,
-  filters: {
-    minCredibility: 0.7,
-    language: 'en'
-  }
-});
-
-// Multi-query search with aggregation
-const aggregatedResults = await searchEngine.multiSearch([
-  'AI in healthcare',
-  'machine learning medical applications',
-  'healthcare automation technology'
-], {
-  aggregate: true,
-  maxResultsPerQuery: 5
-}) as AggregatedResult[];
-
-// Add custom knowledge
-searchEngine.addKnowledge('AI Definition', {
-  term: 'Artificial Intelligence',
-  definition: 'The simulation of human intelligence by machines',
-  categories: ['technology', 'computer science']
-});
-
-// Listen to events
-searchEngine.on('search_complete', (event) => {
-  console.log(`Search completed in ${event.data.processingTime}ms`);
-});
-*/