gthinking 1.0.0

package/search-discovery.ts

@@ -0,0 +1,654 @@
+/**
+ * Search & Discovery Module
+ * Real-time web search with multi-source aggregation and intelligent filtering
+ */
+
+import {
+  SearchQuery,
+  SearchResult,
+  SearchSession,
+  SearchFilters,
+  AggregatedResult,
+  SourceType,
+  ResultMetadata,
+  ConfidenceLevel,
+  ThinkingEvent,
+  ThinkingError,
+  ThinkingStage
+} from './types';
+import { EventEmitter } from 'events';
+
+// ============================================================================
+// SEARCH PROVIDER INTERFACES
+// ============================================================================
+
+interface SearchProvider {
+  name: string;
+  search(query: string, filters: SearchFilters, maxResults: number): Promise<SearchResult[]>;
+  getCredibilityScore(url: string): number;
+}
+
+// ============================================================================
+// WEB SEARCH PROVIDER IMPLEMENTATION
+// ============================================================================
+
+class WebSearchProvider implements SearchProvider {
+  name = 'web_search';
+
+  async search(
+    query: string,
+    filters: SearchFilters,
+    maxResults: number
+  ): Promise<SearchResult[]> {
+    // In real implementation, this would call actual search APIs
+    // For demonstration, we simulate search results
+    const simulatedResults: SearchResult[] = [];
+    
+    // Generate diverse simulated results based on query
+    const topics = this.extractTopics(query);
+    
+    for (let i = 0; i < maxResults; i++) {
+      const topic = topics[i % topics.length];
+      const result: SearchResult = {
+        id: `web_${Date.now()}_${i}`,
+        title: this.generateTitle(topic, i),
+        url: `https://example.com/article/${topic.replace(/\s+/g, '-')}-${i}`,
+        snippet: this.generateSnippet(topic, query),
+        source: SourceType.WEB,
+        credibility: this.calculateCredibility(i),
+        relevance: this.calculateRelevance(query, topic, i),
+        timestamp: new Date(),
+        metadata: this.generateMetadata(topic)
+      };
+      simulatedResults.push(result);
+    }
+
+    return this.applyFilters(simulatedResults, filters);
+  }
+
+  getCredibilityScore(url: string): number {
+    const credibleDomains = [
+      'edu', 'gov', 'ac.uk', 'ac.jp', 'arxiv.org',
+      'nature.com', 'science.org', 'ieee.org', 'acm.org'
+    ];
+    
+    const lowCredibilityDomains = [
+      'blogspot', 'wordpress', 'medium', 'wix', 'weebly'
+    ];
+
+    if (credibleDomains.some(d => url.includes(d))) return 0.9;
+    if (lowCredibilityDomains.some(d => url.includes(d))) return 0.4;
+    return 0.6;
+  }
+
+  private extractTopics(query: string): string[] {
+    // Simple topic extraction - in real implementation, use NLP
+    const commonWords = ['what', 'how', 'why', 'when', 'where', 'who', 'is', 'are', 'the', 'a', 'an'];
+    return query
+      .toLowerCase()
+      .split(/\s+/)
+      .filter(word => !commonWords.includes(word) && word.length > 3);
+  }
+
+  private generateTitle(topic: string, index: number): string {
+    const templates = [
+      `Complete Guide to ${topic}`,
+      `${topic}: Everything You Need to Know`,
+      `Understanding ${topic} in Depth`,
+      `The Future of ${topic}`,
+      `${topic} Best Practices and Strategies`,
+      `Advanced ${topic} Techniques`,
+      `${topic} Case Studies and Examples`,
+      `Breaking Down ${topic}: A Comprehensive Analysis`
+    ];
+    return templates[index % templates.length];
+  }
+
+  private generateSnippet(topic: string, query: string): string {
+    return `This comprehensive resource about ${topic} provides detailed insights related to "${query}". ` +
+           `It covers key concepts, practical applications, and expert perspectives on the subject matter.`;
+  }
+
+  private calculateCredibility(index: number): number {
+    // Higher ranked results typically have higher credibility
+    return Math.max(0.3, 0.95 - (index * 0.05));
+  }
+
+  private calculateRelevance(query: string, topic: string, index: number): number {
+    const queryWords = query.toLowerCase().split(/\s+/);
+    const topicWords = topic.toLowerCase().split(/\s+/);
+    const matchCount = topicWords.filter(w => queryWords.includes(w)).length;
+    const baseRelevance = matchCount / Math.max(queryWords.length, topicWords.length);
+    return Math.min(0.95, baseRelevance + (0.9 - index * 0.1));
+  }
+
+  private generateMetadata(topic: string): ResultMetadata {
+    return {
+      author: `Expert on ${topic}`,
+      publishDate: new Date(Date.now() - Math.random() * 365 * 24 * 60 * 60 * 1000),
+      wordCount: Math.floor(1000 + Math.random() * 4000),
+      readingTime: Math.floor(5 + Math.random() * 20),
+      relatedTopics: [topic, 'related-concept-1', 'related-concept-2'],
+      citations: Math.floor(Math.random() * 50)
+    };
+  }
+
+  private applyFilters(results: SearchResult[], filters: SearchFilters): SearchResult[] {
+    return results.filter(result => {
+      if (filters.minCredibility && result.credibility < filters.minCredibility) {
+        return false;
+      }
+      if (filters.domains && filters.domains.length > 0) {
+        const domainMatch = filters.domains.some(d => result.url.includes(d));
+        if (!domainMatch) return false;
+      }
+      if (filters.excludeDomains) {
+        const excluded = filters.excludeDomains.some(d => result.url.includes(d));
+        if (excluded) return false;
+      }
+      return true;
+    });
+  }
+}
+
+// ============================================================================
+// KNOWLEDGE BASE SEARCH PROVIDER
+// ============================================================================
+
+class KnowledgeBaseProvider implements SearchProvider {
+  name = 'knowledge_base';
+  private knowledgeStore: Map<string, unknown> = new Map();
+
+  async search(
+    query: string,
+    filters: SearchFilters,
+    maxResults: number
+  ): Promise<SearchResult[]> {
+    const results: SearchResult[] = [];
+    const queryLower = query.toLowerCase();
+
+    // Search through stored knowledge
+    for (const [key, value] of this.knowledgeStore.entries()) {
+      if (key.toLowerCase().includes(queryLower)) {
+        results.push({
+          id: `kb_${key}`,
+          title: key,
+          url: `knowledge://${key}`,
+          snippet: JSON.stringify(value).substring(0, 200),
+          source: SourceType.KNOWLEDGE_BASE,
+          credibility: 0.95,
+          relevance: 0.9,
+          timestamp: new Date(),
+          metadata: { wordCount: JSON.stringify(value).length }
+        });
+      }
+      if (results.length >= maxResults) break;
+    }
+
+    return results;
+  }
+
+  getCredibilityScore(): number {
+    return 0.95; // Knowledge base has high credibility
+  }
+
+  addKnowledge(key: string, value: unknown): void {
+    this.knowledgeStore.set(key, value);
+  }
+
+  getKnowledge(key: string): unknown {
+    return this.knowledgeStore.get(key);
+  }
+}
+
+// ============================================================================
+// CACHE SEARCH PROVIDER
+// ============================================================================
+
+class CacheProvider implements SearchProvider {
+  name = 'cache';
+  private cache: Map<string, { result: SearchResult; timestamp: number; ttl: number }> = new Map();
+  private defaultTTL = 3600000; // 1 hour
+
+  async search(
+    query: string,
+    _filters: SearchFilters,
+    maxResults: number
+  ): Promise<SearchResult[]> {
+    const results: SearchResult[] = [];
+    const now = Date.now();
+
+    for (const [key, cached] of this.cache.entries()) {
+      if (key.includes(query) && now - cached.timestamp < cached.ttl) {
+        results.push(cached.result);
+      }
+      if (results.length >= maxResults) break;
+    }
+
+    return results;
+  }
+
+  getCredibilityScore(): number {
+    return 0.8;
+  }
+
+  cacheResult(query: string, result: SearchResult, ttl?: number): void {
+    this.cache.set(query, {
+      result,
+      timestamp: Date.now(),
+      ttl: ttl || this.defaultTTL
+    });
+  }
+
+  invalidate(query: string): void {
+    this.cache.delete(query);
+  }
+
+  clear(): void {
+    this.cache.clear();
+  }
+}
+
+// ============================================================================
+// MAIN SEARCH & DISCOVERY ENGINE
+// ============================================================================
+
+export class SearchDiscoveryEngine extends EventEmitter {
+  private providers: Map<SourceType, SearchProvider> = new Map();
+  private sessions: Map<string, SearchSession> = new Map();
+  private knowledgeBase: KnowledgeBaseProvider;
+  private cache: CacheProvider;
+
+  constructor() {
+    super();
+    this.knowledgeBase = new KnowledgeBaseProvider();
+    this.cache = new CacheProvider();
+    this.registerDefaultProviders();
+  }
+
+  private registerDefaultProviders(): void {
+    this.providers.set(SourceType.WEB, new WebSearchProvider());
+    this.providers.set(SourceType.KNOWLEDGE_BASE, this.knowledgeBase);
+    this.providers.set(SourceType.CACHE, this.cache);
+  }
+
+  /**
+   * Execute a search query across multiple sources
+   */
+  async search(
+    query: string,
+    options: {
+      sources?: SourceType[];
+      filters?: SearchFilters;
+      maxResults?: number;
+      timeout?: number;
+    } = {}
+  ): Promise<SearchResult[]> {
+    const {
+      sources = [SourceType.WEB, SourceType.KNOWLEDGE_BASE, SourceType.CACHE],
+      filters = {},
+      maxResults = 10,
+      timeout = 30000
+    } = options;
+
+    const searchId = this.generateId();
+    const startTime = Date.now();
+
+    this.emit('search_start', {
+      id: searchId,
+      type: 'stage_start',
+      stage: ThinkingStage.SEARCH,
+      timestamp: new Date(),
+      data: { query, sources },
+      sessionId: searchId
+    } as ThinkingEvent);
+
+    try {
+      // Check cache first
+      const cachedResults = await this.searchWithProvider(
+        SourceType.CACHE,
+        query,
+        filters,
+        maxResults
+      );
+
+      if (cachedResults.length > 0) {
+        this.emit('search_complete', {
+          id: searchId,
+          type: 'search_result',
+          stage: ThinkingStage.SEARCH,
+          timestamp: new Date(),
+          data: { results: cachedResults, fromCache: true },
+          sessionId: searchId
+        } as ThinkingEvent);
+        return cachedResults;
+      }
+
+      // Execute searches across all specified sources in parallel
+      const searchPromises = sources
+        .filter(s => s !== SourceType.CACHE) // Already checked cache
+        .map(source => this.searchWithProvider(source, query, filters, maxResults));
+
+      const resultsArrays = await Promise.allSettled(searchPromises);
+      
+      let allResults: SearchResult[] = [];
+      resultsArrays.forEach((result, index) => {
+        if (result.status === 'fulfilled') {
+          allResults = allResults.concat(result.value);
+        } else {
+          console.warn(`Search from ${sources[index]} failed:`, result.reason);
+        }
+      });
+
+      // Rank and filter results
+      const rankedResults = this.rankResults(allResults, query).slice(0, maxResults);
+
+      // Cache top results
+      rankedResults.forEach(result => {
+        this.cache.cacheResult(query, result);
+      });
+
+      const processingTime = Date.now() - startTime;
+
+      this.emit('search_complete', {
+        id: searchId,
+        type: 'search_result',
+        stage: ThinkingStage.SEARCH,
+        timestamp: new Date(),
+        data: { 
+          results: rankedResults, 
+          fromCache: false,
+          processingTime,
+          sourcesQueried: sources.length
+        },
+        sessionId: searchId
+      } as ThinkingEvent);
+
+      return rankedResults;
+
+    } catch (error) {
+      this.emit('search_error', {
+        id: searchId,
+        type: 'stage_error',
+        stage: ThinkingStage.SEARCH,
+        timestamp: new Date(),
+        data: { error },
+        sessionId: searchId
+      } as ThinkingEvent);
+
+      throw new ThinkingError(
+        `Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        ThinkingStage.SEARCH,
+        true,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * Multi-query search for comprehensive coverage
+   */
+  async multiSearch(
+    queries: string[],
+    options: {
+      sources?: SourceType[];
+      filters?: SearchFilters;
+      maxResultsPerQuery?: number;
+      aggregate?: boolean;
+    } = {}
+  ): Promise<SearchResult[] | AggregatedResult[]> {
+    const { aggregate = true, maxResultsPerQuery = 5 } = options;
+
+    const sessionId = this.generateId();
+    const session: SearchSession = {
+      id: sessionId,
+      queries: [],
+      results: [],
+      aggregatedResults: [],
+      startTime: new Date()
+    };
+
+    this.sessions.set(sessionId, session);
+
+    // Execute all queries
+    const searchPromises = queries.map(async (query) => {
+      const results = await this.search(query, {
+        ...options,
+        maxResults: maxResultsPerQuery
+      });
+      return { query, results };
+    });
+
+    const searchResults = await Promise.all(searchPromises);
+
+    // Collect all results
+    searchResults.forEach(({ query, results }) => {
+      session.queries.push({
+        id: this.generateId(),
+        query,
+        sources: options.sources || [SourceType.WEB],
+        filters: options.filters || {},
+        maxResults: maxResultsPerQuery,
+        timeout: 30000,
+        timestamp: new Date()
+      });
+      session.results.push(...results);
+    });
+
+    if (aggregate) {
+      session.aggregatedResults = this.aggregateResults(session.results);
+      return session.aggregatedResults;
+    }
+
+    return session.results;
+  }
+
+  /**
+   * Aggregate results by topic and identify consensus/conflicts
+   */
+  aggregateResults(results: SearchResult[]): AggregatedResult[] {
+    const topicMap = new Map<string, SearchResult[]>();
+
+    // Group by topic (simplified - in real implementation use NLP clustering)
+    results.forEach(result => {
+      const topic = this.extractMainTopic(result.title);
+      if (!topicMap.has(topic)) {
+        topicMap.set(topic, []);
+      }
+      topicMap.get(topic)!.push(result);
+    });
+
+    const aggregated: AggregatedResult[] = [];
+
+    topicMap.forEach((topicResults, topic) => {
+      // Calculate consensus score
+      const avgCredibility = topicResults.reduce((sum, r) => sum + r.credibility, 0) / topicResults.length;
+      const avgRelevance = topicResults.reduce((sum, r) => sum + r.relevance, 0) / topicResults.length;
+      const consensusScore = (avgCredibility + avgRelevance) / 2;
+
+      // Check for conflicting information
+      const conflictingInfo = this.detectConflicts(topicResults);
+
+      // Extract key insights
+      const keyInsights = this.extractKeyInsights(topicResults);
+
+      aggregated.push({
+        topic,
+        results: topicResults,
+        consensusScore,
+        conflictingInfo,
+        keyInsights
+      });
+    });
+
+    // Sort by consensus score
+    return aggregated.sort((a, b) => b.consensusScore - a.consensusScore);
+  }
+
+  /**
+   * Rank results based on multiple factors
+   */
+  private rankResults(results: SearchResult[], query: string): SearchResult[] {
+    return results
+      .map(result => ({
+        ...result,
+        // Calculate composite score
+        compositeScore: this.calculateCompositeScore(result, query)
+      }))
+      .sort((a, b) => (b as any).compositeScore - (a as any).compositeScore);
+  }
+
+  private calculateCompositeScore(result: SearchResult, query: string): number {
+    const relevanceWeight = 0.4;
+    const credibilityWeight = 0.3;
+    const freshnessWeight = 0.15;
+    const engagementWeight = 0.15;
+
+    // Relevance score
+    const relevanceScore = result.relevance;
+
+    // Credibility score
+    const credibilityScore = result.credibility;
+
+    // Freshness score (newer is better)
+    const ageInDays = (Date.now() - result.timestamp.getTime()) / (1000 * 60 * 60 * 24);
+    const freshnessScore = Math.max(0, 1 - ageInDays / 365);
+
+    // Engagement score (based on citations, reading time)
+    const engagementScore = Math.min(1, (result.metadata.citations || 0) / 100);
+
+    return (
+      relevanceScore * relevanceWeight +
+      credibilityScore * credibilityWeight +
+      freshnessScore * freshnessWeight +
+      engagementScore * engagementWeight
+    );
+  }
+
+  /**
+   * Search with a specific provider
+   */
+  private async searchWithProvider(
+    source: SourceType,
+    query: string,
+    filters: SearchFilters,
+    maxResults: number
+  ): Promise<SearchResult[]> {
+    const provider = this.providers.get(source);
+    if (!provider) {
+      throw new Error(`No provider registered for source: ${source}`);
+    }
+    return provider.search(query, filters, maxResults);
+  }
+
+  /**
+   * Add custom search provider
+   */
+  registerProvider(source: SourceType, provider: SearchProvider): void {
+    this.providers.set(source, provider);
+  }
+
+  /**
+   * Add knowledge to knowledge base
+   */
+  addKnowledge(key: string, value: unknown): void {
+    this.knowledgeBase.addKnowledge(key, value);
+  }
+
+  /**
+   * Get search session by ID
+   */
+  getSession(sessionId: string): SearchSession | undefined {
+    return this.sessions.get(sessionId);
+  }
+
+  /**
+   * Clear all sessions
+   */
+  clearSessions(): void {
+    this.sessions.clear();
+  }
+
+  // Helper methods
+  private generateId(): string {
+    return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  private extractMainTopic(title: string): string {
+    // Simplified topic extraction
+    const words = title.toLowerCase().split(/\s+/);
+    const significantWords = words.filter(w => w.length > 4);
+    return significantWords.slice(0, 3).join(' ') || 'general';
+  }
+
+  private detectConflicts(results: SearchResult[]): boolean {
+    // Simplified conflict detection
+    // In real implementation, use NLP to compare content
+    const snippets = results.map(r => r.snippet.toLowerCase());
+    const contradictoryTerms = ['not', 'never', 'false', 'incorrect', 'myth'];
+    
+    return snippets.some(snippet => 
+      contradictoryTerms.some(term => snippet.includes(term))
+    );
+  }
+
+  private extractKeyInsights(results: SearchResult[]): string[] {
+    // Simplified insight extraction
+    const insights: string[] = [];
+    const seenTopics = new Set<string>();
+
+    results.forEach(result => {
+      const topic = this.extractMainTopic(result.title);
+      if (!seenTopics.has(topic)) {
+        seenTopics.add(topic);
+        insights.push(`${topic}: ${result.snippet.substring(0, 100)}...`);
+      }
+    });
+
+    return insights.slice(0, 5);
+  }
+}
+
+// ============================================================================
+// EXPORT SINGLETON INSTANCE
+// ============================================================================
+
+export const searchEngine = new SearchDiscoveryEngine();
+
+// ============================================================================
+// EXAMPLE USAGE
+// ============================================================================
+
+/*
+// Basic search
+const results = await searchEngine.search('artificial intelligence trends 2024', {
+  sources: [SourceType.WEB, SourceType.KNOWLEDGE_BASE],
+  maxResults: 10,
+  filters: {
+    minCredibility: 0.7,
+    language: 'en'
+  }
+});
+
+// Multi-query search with aggregation
+const aggregatedResults = await searchEngine.multiSearch([
+  'AI in healthcare',
+  'machine learning medical applications',
+  'healthcare automation technology'
+], {
+  aggregate: true,
+  maxResultsPerQuery: 5
+}) as AggregatedResult[];
+
+// Add custom knowledge
+searchEngine.addKnowledge('AI Definition', {
+  term: 'Artificial Intelligence',
+  definition: 'The simulation of human intelligence by machines',
+  categories: ['technology', 'computer science']
+});
+
+// Listen to events
+searchEngine.on('search_complete', (event) => {
+  console.log(`Search completed in ${event.data.processingTime}ms`);
+});
+*/

package/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "rootDir": ".",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "node"
+  },
+  "include": [
+    "*.ts"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist"
+  ]
+}