@sesamespace/hivemind 0.8.10 → 0.8.11

package/src/memory/processors/research-digester.ts

@@ -0,0 +1,204 @@
+/**
+ * Research Digester - Extracts and indexes key information from web pages and documents
+ */
+
+import { BackgroundProcessor } from './background-processor';
+import { EventEmitter } from 'events';
+import * as crypto from 'crypto';
+
+export interface ResearchEntry {
+  id: string;
+  url?: string;
+  title: string;
+  summary: string;
+  keyPoints: string[];
+  relatedTopics: string[];
+  timestamp: Date;
+  sourceType: 'web' | 'pdf' | 'markdown' | 'other';
+  content?: string; // Full content for reference
+}
+
+export class ResearchDigester extends BackgroundProcessor {
+  private research: Map<string, ResearchEntry> = new Map();
+  private topicIndex: Map<string, Set<string>> = new Map(); // topic -> research IDs
+  private recentQueue: string[] = []; // URLs/content to process
+
+  constructor() {
+    super('research-digester', 30000); // Process every 30 seconds
+  }
+
+  async process(): Promise<void> {
+    // Process any queued research
+    while (this.recentQueue.length > 0) {
+      const item = this.recentQueue.shift();
+      if (item) {
+        await this.digestContent(item);
+      }
+    }
+
+    // Emit current research state
+    this.emit('research-updated', {
+      totalEntries: this.research.size,
+      topics: Array.from(this.topicIndex.keys())
+    });
+  }
+
+  /**
+   * Queue content for digestion
+   */
+  async addContent(content: string, metadata: {
+    url?: string;
+    title?: string;
+    sourceType?: ResearchEntry['sourceType'];
+  } = {}): Promise<void> {
+    this.recentQueue.push(JSON.stringify({ content, metadata }));
+  }
+
+  private async digestContent(item: string): Promise<void> {
+    try {
+      const { content, metadata } = JSON.parse(item);
+      
+      // Extract key information
+      const entry: ResearchEntry = {
+        id: crypto.randomBytes(8).toString('hex'),
+        url: metadata.url,
+        title: metadata.title || this.extractTitle(content),
+        summary: this.extractSummary(content),
+        keyPoints: this.extractKeyPoints(content),
+        relatedTopics: this.extractTopics(content),
+        timestamp: new Date(),
+        sourceType: metadata.sourceType || 'other',
+        content: content.substring(0, 5000) // Store first 5k chars
+      };
+
+      // Store the entry
+      this.research.set(entry.id, entry);
+
+      // Index by topics
+      for (const topic of entry.relatedTopics) {
+        if (!this.topicIndex.has(topic)) {
+          this.topicIndex.set(topic, new Set());
+        }
+        this.topicIndex.get(topic)!.add(entry.id);
+      }
+
+      this.emit('research-digested', entry);
+    } catch (error) {
+      this.emit('error', { error, item });
+    }
+  }
+
+  private extractTitle(content: string): string {
+    // Look for markdown headers or first significant line
+    const lines = content.split('\n');
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('#')) {
+        return trimmed.replace(/^#+\s*/, '');
+      }
+      if (trimmed.length > 10 && trimmed.length < 100) {
+        return trimmed;
+      }
+    }
+    return 'Untitled Research';
+  }
+
+  private extractSummary(content: string): string {
+    // Get first meaningful paragraph
+    const paragraphs = content.split(/\n\n+/);
+    for (const para of paragraphs) {
+      const trimmed = para.trim();
+      if (trimmed.length > 50 && !trimmed.startsWith('#')) {
+        return trimmed.substring(0, 200) + '...';
+      }
+    }
+    return content.substring(0, 200) + '...';
+  }
+
+  private extractKeyPoints(content: string): string[] {
+    const keyPoints: string[] = [];
+    const lines = content.split('\n');
+    
+    for (const line of lines) {
+      const trimmed = line.trim();
+      // Look for bullet points, numbered lists, or key phrases
+      if (trimmed.match(/^[-*•]\s+/) || trimmed.match(/^\d+\.\s+/)) {
+        keyPoints.push(trimmed.replace(/^[-*•\d.]\s+/, ''));
+      }
+      // Look for sentences with key indicator words
+      if (trimmed.match(/\b(important|key|critical|essential|must|should|note)\b/i)) {
+        keyPoints.push(trimmed);
+      }
+    }
+    
+    // Limit to top 10 key points
+    return keyPoints.slice(0, 10);
+  }
+
+  private extractTopics(content: string): string[] {
+    const topics = new Set<string>();
+    
+    // Common technical topics
+    const topicPatterns = [
+      /\b(typescript|javascript|python|rust|go)\b/gi,
+      /\b(react|vue|angular|svelte)\b/gi,
+      /\b(docker|kubernetes|k8s)\b/gi,
+      /\b(api|rest|graphql|grpc)\b/gi,
+      /\b(database|sql|nosql|postgres|mysql|mongodb)\b/gi,
+      /\b(ai|ml|machine learning|llm|gpt)\b/gi,
+      /\b(memory|context|embedding|vector)\b/gi,
+      /\b(git|github|version control)\b/gi,
+      /\b(testing|ci|cd|deployment)\b/gi,
+      /\b(security|auth|authentication|authorization)\b/gi
+    ];
+
+    for (const pattern of topicPatterns) {
+      const matches = content.match(pattern);
+      if (matches) {
+        matches.forEach(match => topics.add(match.toLowerCase()));
+      }
+    }
+
+    return Array.from(topics).slice(0, 5); // Top 5 topics
+  }
+
+  /**
+   * Search research by topic or content
+   */
+  async search(query: string): Promise<ResearchEntry[]> {
+    const results: ResearchEntry[] = [];
+    const queryLower = query.toLowerCase();
+
+    // Search by topic
+    if (this.topicIndex.has(queryLower)) {
+      const ids = this.topicIndex.get(queryLower)!;
+      for (const id of ids) {
+        const entry = this.research.get(id);
+        if (entry) results.push(entry);
+      }
+    }
+
+    // Search in content
+    for (const entry of this.research.values()) {
+      if (entry.title.toLowerCase().includes(queryLower) ||
+          entry.summary.toLowerCase().includes(queryLower) ||
+          entry.keyPoints.some(kp => kp.toLowerCase().includes(queryLower))) {
+        if (!results.includes(entry)) {
+          results.push(entry);
+        }
+      }
+    }
+
+    // Sort by recency
+    return results.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+  }
+
+  /**
+   * Get recent research entries
+   */
+  async getRecent(limit: number = 10): Promise<ResearchEntry[]> {
+    return Array.from(this.research.values())
+      .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime())
+      .slice(0, limit);
+  }
+}