crawlforge-mcp-server 3.0.0

package/src/core/llm/LLMManager.js

@@ -0,0 +1,415 @@
+import { OpenAIProvider } from './OpenAIProvider.js';
+import { AnthropicProvider } from './AnthropicProvider.js';
+import { Logger } from '../../utils/Logger.js';
+
+/**
+ * LLM Manager
+ * Manages multiple LLM providers and provides unified interface
+ */
+export class LLMManager {
+  constructor(options = {}) {
+    this.logger = new Logger({ component: 'LLMManager' });
+    this.providers = new Map();
+    this.defaultProvider = null;
+    this.fallbackProvider = null;
+    
+    this.initializeProviders(options);
+  }
+
+  /**
+   * Initialize available LLM providers
+   */
+  initializeProviders(options) {
+    const {
+      openai = {},
+      anthropic = {},
+      defaultProvider = 'auto'
+    } = options;
+
+    // Initialize OpenAI provider
+    if (openai.apiKey || process.env.OPENAI_API_KEY) {
+      const openaiProvider = new OpenAIProvider(openai);
+      this.providers.set('openai', openaiProvider);
+      this.logger.info('OpenAI provider initialized');
+    }
+
+    // Initialize Anthropic provider
+    if (anthropic.apiKey || process.env.ANTHROPIC_API_KEY) {
+      const anthropicProvider = new AnthropicProvider(anthropic);
+      this.providers.set('anthropic', anthropicProvider);
+      this.logger.info('Anthropic provider initialized');
+    }
+
+    // Set default provider
+    this.setDefaultProvider(defaultProvider);
+  }
+
+  /**
+   * Set the default provider
+   */
+  setDefaultProvider(providerName) {
+    if (providerName === 'auto') {
+      // Auto-select: prefer OpenAI for embeddings, fallback to Anthropic
+      if (this.providers.has('openai')) {
+        this.defaultProvider = 'openai';
+        this.fallbackProvider = this.providers.has('anthropic') ? 'anthropic' : null;
+      } else if (this.providers.has('anthropic')) {
+        this.defaultProvider = 'anthropic';
+        this.fallbackProvider = null;
+      }
+    } else if (this.providers.has(providerName)) {
+      this.defaultProvider = providerName;
+      // Set fallback to other available provider
+      for (const [name, provider] of this.providers) {
+        if (name !== providerName) {
+          this.fallbackProvider = name;
+          break;
+        }
+      }
+    }
+
+    if (this.defaultProvider) {
+      this.logger.info(`Default LLM provider set to: ${this.defaultProvider}`);
+      if (this.fallbackProvider) {
+        this.logger.info(`Fallback LLM provider: ${this.fallbackProvider}`);
+      }
+    } else {
+      this.logger.warn('No LLM providers available');
+    }
+  }
+
+  /**
+   * Get a provider instance
+   */
+  getProvider(name = null) {
+    const providerName = name || this.defaultProvider;
+    if (!providerName) {
+      throw new Error('No LLM provider available');
+    }
+    
+    const provider = this.providers.get(providerName);
+    if (!provider) {
+      throw new Error(`LLM provider '${providerName}' not found`);
+    }
+    
+    return provider;
+  }
+
+  /**
+   * Generate completion with fallback support
+   */
+  async generateCompletion(prompt, options = {}) {
+    const { provider = null, ...llmOptions } = options;
+    
+    try {
+      const llmProvider = this.getProvider(provider);
+      return await llmProvider.generateCompletion(prompt, llmOptions);
+    } catch (error) {
+      this.logger.warn(`Primary provider failed: ${error.message}`);
+      
+      // Try fallback provider if available
+      if (this.fallbackProvider && (!provider || provider === this.defaultProvider)) {
+        try {
+          this.logger.info(`Trying fallback provider: ${this.fallbackProvider}`);
+          const fallbackLLM = this.getProvider(this.fallbackProvider);
+          return await fallbackLLM.generateCompletion(prompt, llmOptions);
+        } catch (fallbackError) {
+          this.logger.error(`Fallback provider also failed: ${fallbackError.message}`);
+        }
+      }
+      
+      throw error;
+    }
+  }
+
+  /**
+   * Generate embeddings with fallback support
+   */
+  async generateEmbedding(text, options = {}) {
+    const { provider = null } = options;
+    
+    try {
+      const llmProvider = this.getProvider(provider);
+      return await llmProvider.generateEmbedding(text);
+    } catch (error) {
+      this.logger.warn(`Primary provider embedding failed: ${error.message}`);
+      
+      // Try fallback provider if available
+      if (this.fallbackProvider && (!provider || provider === this.defaultProvider)) {
+        try {
+          this.logger.info(`Trying fallback provider for embedding: ${this.fallbackProvider}`);
+          const fallbackLLM = this.getProvider(this.fallbackProvider);
+          return await fallbackLLM.generateEmbedding(text);
+        } catch (fallbackError) {
+          this.logger.error(`Fallback provider embedding also failed: ${fallbackError.message}`);
+        }
+      }
+      
+      throw error;
+    }
+  }
+
+  /**
+   * Calculate semantic similarity
+   */
+  async calculateSimilarity(text1, text2, options = {}) {
+    const { provider = null } = options;
+    
+    try {
+      const llmProvider = this.getProvider(provider);
+      return await llmProvider.calculateSimilarity(text1, text2);
+    } catch (error) {
+      this.logger.warn(`Primary provider similarity failed: ${error.message}`);
+      
+      // Try fallback provider if available
+      if (this.fallbackProvider && (!provider || provider === this.defaultProvider)) {
+        try {
+          this.logger.info(`Trying fallback provider for similarity: ${this.fallbackProvider}`);
+          const fallbackLLM = this.getProvider(this.fallbackProvider);
+          return await fallbackLLM.calculateSimilarity(text1, text2);
+        } catch (fallbackError) {
+          this.logger.error(`Fallback provider similarity also failed: ${fallbackError.message}`);
+        }
+      }
+      
+      throw error;
+    }
+  }
+
+  /**
+   * Generate query expansion suggestions
+   */
+  async expandQuery(query, options = {}) {
+    const {
+      maxExpansions = 5,
+      includeContextual = true,
+      includeSynonyms = true,
+      includeRelated = true
+    } = options;
+
+    const systemPrompt = `You are a query expansion expert. Generate relevant search query variations for research purposes.
+
+Rules:
+1. Return only the query variations, one per line
+2. Focus on research-oriented variations
+3. Include different perspectives and angles
+4. Maintain semantic relevance
+5. Keep queries concise and searchable
+6. Maximum ${maxExpansions} variations`;
+
+    let prompt = `Original query: "${query}"\n\nGenerate ${maxExpansions} research-focused query variations:`;
+
+    if (includeContextual) {
+      prompt += '\n- Include contextual variations';
+    }
+    if (includeSynonyms) {
+      prompt += '\n- Include synonym-based variations';
+    }
+    if (includeRelated) {
+      prompt += '\n- Include related concept variations';
+    }
+
+    try {
+      const response = await this.generateCompletion(prompt, {
+        systemPrompt,
+        maxTokens: 300,
+        temperature: 0.8
+      });
+
+      return response
+        .split('\n')
+        .map(line => line.trim())
+        .filter(line => line && !line.startsWith('-') && !line.includes(':'))
+        .slice(0, maxExpansions);
+    } catch (error) {
+      this.logger.warn('LLM query expansion failed, using fallback', { error: error.message });
+      return this.fallbackQueryExpansion(query, maxExpansions);
+    }
+  }
+
+  /**
+   * Analyze content relevance to a topic
+   */
+  async analyzeRelevance(content, topic, options = {}) {
+    const { maxContentLength = 2000 } = options;
+    
+    const truncatedContent = content.length > maxContentLength 
+      ? content.substring(0, maxContentLength) + '...'
+      : content;
+
+    const systemPrompt = `You are a content relevance analyzer. Evaluate how relevant the given content is to the specified research topic.
+
+Return a JSON object with:
+{
+  "relevanceScore": 0.0-1.0,
+  "keyPoints": ["point1", "point2", ...],
+  "topicAlignment": "description of alignment",
+  "credibilityIndicators": ["indicator1", "indicator2", ...]
+}`;
+
+    const prompt = `Research Topic: "${topic}"
+
+Content to analyze:
+${truncatedContent}
+
+Analyze the relevance of this content to the research topic:`;
+
+    try {
+      const response = await this.generateCompletion(prompt, {
+        systemPrompt,
+        maxTokens: 500,
+        temperature: 0.3
+      });
+
+      const analysis = JSON.parse(response);
+      return {
+        relevanceScore: Math.max(0, Math.min(1, analysis.relevanceScore || 0.5)),
+        keyPoints: analysis.keyPoints || [],
+        topicAlignment: analysis.topicAlignment || '',
+        credibilityIndicators: analysis.credibilityIndicators || []
+      };
+    } catch (error) {
+      this.logger.warn('LLM relevance analysis failed, using fallback', { error: error.message });
+      return this.fallbackRelevanceAnalysis(content, topic);
+    }
+  }
+
+  /**
+   * Generate research synthesis
+   */
+  async synthesizeFindings(findings, topic, options = {}) {
+    const { maxFindings = 10, includeConflicts = true } = options;
+    
+    const limitedFindings = findings.slice(0, maxFindings);
+    
+    const systemPrompt = `You are a research synthesis expert. Create a comprehensive synthesis of research findings on a given topic.
+
+Generate a JSON response with:
+{
+  "summary": "overall summary",
+  "keyInsights": ["insight1", "insight2", ...],
+  "themes": ["theme1", "theme2", ...],
+  "confidence": 0.0-1.0,
+  "gaps": ["gap1", "gap2", ...],
+  "recommendations": ["rec1", "rec2", ...]
+}`;
+
+    const findingsText = limitedFindings
+      .map((finding, index) => `${index + 1}. ${finding.finding || finding.text || finding}`)
+      .join('\n');
+
+    const prompt = `Research Topic: "${topic}"
+
+Research Findings:
+${findingsText}
+
+Synthesize these findings into a comprehensive analysis:`;
+
+    try {
+      const response = await this.generateCompletion(prompt, {
+        systemPrompt,
+        maxTokens: 800,
+        temperature: 0.4
+      });
+
+      return JSON.parse(response);
+    } catch (error) {
+      this.logger.warn('LLM synthesis failed, using fallback', { error: error.message });
+      return this.fallbackSynthesis(findings, topic);
+    }
+  }
+
+  /**
+   * Fallback query expansion without LLM
+   */
+  fallbackQueryExpansion(query, maxExpansions) {
+    const variations = [];
+    const words = query.toLowerCase().split(/\s+/);
+    
+    // Question variations
+    variations.push(`what is ${query}`);
+    variations.push(`how does ${query} work`);
+    variations.push(`${query} research`);
+    variations.push(`${query} analysis`);
+    variations.push(`latest ${query}`);
+    
+    return variations.slice(0, maxExpansions);
+  }
+
+  /**
+   * Fallback relevance analysis without LLM
+   */
+  fallbackRelevanceAnalysis(content, topic) {
+    const topicWords = topic.toLowerCase().split(/\s+/);
+    const contentWords = content.toLowerCase().split(/\s+/);
+    
+    const matches = topicWords.filter(word => 
+      contentWords.some(cWord => cWord.includes(word) || word.includes(cWord))
+    );
+    
+    const relevanceScore = matches.length / topicWords.length;
+    
+    return {
+      relevanceScore: Math.min(1, relevanceScore),
+      keyPoints: [content.substring(0, 100) + '...'],
+      topicAlignment: `Found ${matches.length}/${topicWords.length} topic keywords`,
+      credibilityIndicators: []
+    };
+  }
+
+  /**
+   * Fallback synthesis without LLM
+   */
+  fallbackSynthesis(findings, topic) {
+    return {
+      summary: `Collected ${findings.length} findings related to ${topic}`,
+      keyInsights: findings.slice(0, 3).map(f => f.finding || f.text || f),
+      themes: ['general research'],
+      confidence: 0.5,
+      gaps: ['Limited synthesis without LLM'],
+      recommendations: ['Use LLM provider for detailed synthesis']
+    };
+  }
+
+  /**
+   * Check if any LLM provider is available
+   */
+  isAvailable() {
+    return this.providers.size > 0;
+  }
+
+  /**
+   * Get available providers metadata
+   */
+  getProvidersMetadata() {
+    const metadata = {};
+    for (const [name, provider] of this.providers) {
+      metadata[name] = provider.getMetadata();
+    }
+    return metadata;
+  }
+
+  /**
+   * Health check for all providers
+   */
+  async healthCheck() {
+    const health = {};
+    
+    for (const [name, provider] of this.providers) {
+      try {
+        const isAvailable = await provider.isAvailable();
+        health[name] = {
+          available: isAvailable,
+          metadata: provider.getMetadata()
+        };
+      } catch (error) {
+        health[name] = {
+          available: false,
+          error: error.message
+        };
+      }
+    }
+    
+    return health;
+  }
+}

package/src/core/llm/LLMProvider.js

@@ -0,0 +1,97 @@
+import { Logger } from '../../utils/Logger.js';
+
+/**
+ * Base LLM Provider class
+ * Defines the interface that all LLM providers must implement
+ */
+export class LLMProvider {
+  constructor(options = {}) {
+    this.logger = new Logger({ component: 'LLMProvider' });
+    this.config = options;
+  }
+
+  /**
+   * Generate a completion from the LLM
+   * @param {string} prompt - The input prompt
+   * @param {Object} options - Generation options
+   * @returns {Promise<string>} Generated text
+   */
+  async generateCompletion(prompt, options = {}) {
+    throw new Error('generateCompletion must be implemented by subclass');
+  }
+
+  /**
+   * Generate embeddings for semantic similarity
+   * @param {string} text - Text to embed
+   * @returns {Promise<number[]>} Embedding vector
+   */
+  async generateEmbedding(text) {
+    throw new Error('generateEmbedding must be implemented by subclass');
+  }
+
+  /**
+   * Calculate semantic similarity between two texts
+   * @param {string} text1 - First text
+   * @param {string} text2 - Second text
+   * @returns {Promise<number>} Similarity score (0-1)
+   */
+  async calculateSimilarity(text1, text2) {
+    const embedding1 = await this.generateEmbedding(text1);
+    const embedding2 = await this.generateEmbedding(text2);
+    return this.cosineSimilarity(embedding1, embedding2);
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   * @param {number[]} vec1 - First vector
+   * @param {number[]} vec2 - Second vector
+   * @returns {number} Similarity score (0-1)
+   */
+  cosineSimilarity(vec1, vec2) {
+    if (vec1.length !== vec2.length) {
+      throw new Error('Vectors must have the same length');
+    }
+
+    let dotProduct = 0;
+    let norm1 = 0;
+    let norm2 = 0;
+
+    for (let i = 0; i < vec1.length; i++) {
+      dotProduct += vec1[i] * vec2[i];
+      norm1 += vec1[i] * vec1[i];
+      norm2 += vec2[i] * vec2[i];
+    }
+
+    const magnitude = Math.sqrt(norm1) * Math.sqrt(norm2);
+    return magnitude > 0 ? dotProduct / magnitude : 0;
+  }
+
+  /**
+   * Check if the provider is available and configured
+   * @returns {Promise<boolean>} True if available
+   */
+  async isAvailable() {
+    try {
+      await this.generateCompletion('test', { maxTokens: 1 });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  /**
+   * Get provider metadata
+   * @returns {Object} Provider information
+   */
+  getMetadata() {
+    return {
+      name: this.constructor.name,
+      config: this.config,
+      capabilities: {
+        completion: true,
+        embedding: false,
+        similarity: false
+      }
+    };
+  }
+}

package/src/core/llm/OpenAIProvider.js

@@ -0,0 +1,127 @@
+import { LLMProvider } from './LLMProvider.js';
+
+/**
+ * OpenAI API Provider
+ * Implements LLM operations using OpenAI's GPT models
+ */
+export class OpenAIProvider extends LLMProvider {
+  constructor(options = {}) {
+    super(options);
+    
+    this.apiKey = options.apiKey || process.env.OPENAI_API_KEY;
+    this.baseURL = options.baseURL || 'https://api.openai.com/v1';
+    this.model = options.model || 'gpt-3.5-turbo';
+    this.embeddingModel = options.embeddingModel || 'text-embedding-ada-002';
+    this.timeout = options.timeout || 30000;
+    
+    if (!this.apiKey) {
+      this.logger.warn('OpenAI API key not configured');
+    }
+  }
+
+  async generateCompletion(prompt, options = {}) {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    const {
+      maxTokens = 1000,
+      temperature = 0.7,
+      systemPrompt = null,
+      stopSequences = null
+    } = options;
+
+    const messages = [];
+    
+    if (systemPrompt) {
+      messages.push({ role: 'system', content: systemPrompt });
+    }
+    
+    messages.push({ role: 'user', content: prompt });
+
+    try {
+      const response = await fetch(`${this.baseURL}/chat/completions`, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages,
+          max_tokens: maxTokens,
+          temperature,
+          stop: stopSequences
+        }),
+        signal: AbortSignal.timeout(this.timeout)
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json().catch(() => ({}));
+        throw new Error(`OpenAI API error: ${response.status} - ${errorData.error?.message || response.statusText}`);
+      }
+
+      const data = await response.json();
+      
+      if (!data.choices || data.choices.length === 0) {
+        throw new Error('No completion generated');
+      }
+
+      return data.choices[0].message.content.trim();
+    } catch (error) {
+      this.logger.error('OpenAI completion failed', { error: error.message });
+      throw error;
+    }
+  }
+
+  async generateEmbedding(text) {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    try {
+      const response = await fetch(`${this.baseURL}/embeddings`, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          model: this.embeddingModel,
+          input: text
+        }),
+        signal: AbortSignal.timeout(this.timeout)
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json().catch(() => ({}));
+        throw new Error(`OpenAI API error: ${response.status} - ${errorData.error?.message || response.statusText}`);
+      }
+
+      const data = await response.json();
+      
+      if (!data.data || data.data.length === 0) {
+        throw new Error('No embedding generated');
+      }
+
+      return data.data[0].embedding;
+    } catch (error) {
+      this.logger.error('OpenAI embedding failed', { error: error.message });
+      throw error;
+    }
+  }
+
+  getMetadata() {
+    return {
+      ...super.getMetadata(),
+      name: 'OpenAI',
+      model: this.model,
+      embeddingModel: this.embeddingModel,
+      capabilities: {
+        completion: true,
+        embedding: true,
+        similarity: true
+      }
+    };
+  }
+}