@sesamespace/hivemind 0.10.0 → 0.11.0

package/src/memory/dashboard-integration.ts

@@ -0,0 +1,295 @@
+/**
+ * Dashboard Integration - Exposes memory system state to the Hivemind dashboard
+ */
+
+import { EventEmitter } from 'events';
+import express from 'express';
+import { MessageFlowIntegration } from './processors/message-flow-integration';
+import { ResearchDigester } from './processors/research-digester';
+import { CommandLearner } from './processors/command-learner';
+import { AgentSync } from './processors/agent-sync';
+
+export interface DashboardConfig {
+  port?: number;
+  messageFlow: MessageFlowIntegration;
+  researchDigester?: ResearchDigester;
+  commandLearner?: CommandLearner;
+  agentSync?: AgentSync;
+}
+
+export class DashboardIntegration extends EventEmitter {
+  private app: express.Application;
+  private messageFlow: MessageFlowIntegration;
+  private researchDigester?: ResearchDigester;
+  private commandLearner?: CommandLearner;
+  private agentSync?: AgentSync;
+
+  constructor(config: DashboardConfig) {
+    super();
+    
+    this.messageFlow = config.messageFlow;
+    this.researchDigester = config.researchDigester;
+    this.commandLearner = config.commandLearner;
+    this.agentSync = config.agentSync;
+
+    this.app = express();
+    this.setupRoutes();
+  }
+
+  private setupRoutes(): void {
+    this.app.use(express.json());
+
+    // Memory system overview
+    this.app.get('/api/memory/overview', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        const overview = {
+          ...state,
+          research: this.researchDigester ? {
+            entries: (await this.researchDigester['research']).size,
+            topics: Array.from(this.researchDigester['topicIndex'].keys())
+          } : null,
+          commands: this.commandLearner ? {
+            patterns: this.commandLearner['patterns'].size,
+            categories: Array.from(this.commandLearner['categoryIndex'].keys())
+          } : null,
+          agents: this.agentSync ? {
+            known: (await this.agentSync.getAgentKnowledge()).length,
+            sharedTasks: (await this.agentSync.getSharedTasks()).length
+          } : null
+        };
+        res.json(overview);
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Working set - currently active code files
+    this.app.get('/api/memory/working-set', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        res.json({
+          files: state.workingSet,
+          totalFiles: state.workingSet.length
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Active tasks
+    this.app.get('/api/memory/tasks', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        const sharedTasks = this.agentSync ? await this.agentSync.getSharedTasks() : [];
+        
+        res.json({
+          localTasks: state.tasks,
+          sharedTasks: sharedTasks,
+          total: state.tasks.length + sharedTasks.length
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Research entries
+    this.app.get('/api/memory/research', async (req, res) => {
+      if (!this.researchDigester) {
+        return res.json({ entries: [], topics: [] });
+      }
+
+      try {
+        const recent = await this.researchDigester.getRecent(20);
+        res.json({
+          entries: recent,
+          topics: Array.from(this.researchDigester['topicIndex'].keys())
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Search research
+    this.app.get('/api/memory/research/search', async (req, res) => {
+      if (!this.researchDigester) {
+        return res.json({ results: [] });
+      }
+
+      try {
+        const { q } = req.query;
+        const results = await this.researchDigester.search(q as string);
+        res.json({ results });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Command patterns
+    this.app.get('/api/memory/commands', async (req, res) => {
+      if (!this.commandLearner) {
+        return res.json({ patterns: [], categories: [] });
+      }
+
+      try {
+        const patterns = Array.from(this.commandLearner['patterns'].values());
+        const categories = Array.from(this.commandLearner['categoryIndex'].keys());
+        
+        res.json({
+          patterns: patterns.sort((a, b) => b.usageCount - a.usageCount),
+          categories
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Command suggestions
+    this.app.get('/api/memory/commands/suggest', async (req, res) => {
+      if (!this.commandLearner) {
+        return res.json({ suggestions: [] });
+      }
+
+      try {
+        const { context, category } = req.query;
+        const suggestions = await this.commandLearner.getSuggestions(
+          context as string || '',
+          category as string
+        );
+        res.json({ suggestions });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Agent knowledge
+    this.app.get('/api/memory/agents', async (req, res) => {
+      if (!this.agentSync) {
+        return res.json({ agents: [] });
+      }
+
+      try {
+        const agents = await this.agentSync.getAgentKnowledge();
+        res.json({ agents });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Memory timeline - recent activity
+    this.app.get('/api/memory/timeline', async (req, res) => {
+      try {
+        const timeline = [];
+        
+        // Add recent tasks
+        const state = await this.messageFlow.getState();
+        for (const task of state.tasks) {
+          timeline.push({
+            type: 'task',
+            timestamp: task.lastUpdate,
+            description: task.description,
+            metadata: { state: task.state }
+          });
+        }
+
+        // Add recent research
+        if (this.researchDigester) {
+          const research = await this.researchDigester.getRecent(5);
+          for (const entry of research) {
+            timeline.push({
+              type: 'research',
+              timestamp: entry.timestamp,
+              description: entry.title,
+              metadata: { url: entry.url, topics: entry.relatedTopics }
+            });
+          }
+        }
+
+        // Add recent commands
+        if (this.commandLearner) {
+          const patterns = Array.from(this.commandLearner['patterns'].values())
+            .sort((a, b) => b.lastUsed.getTime() - a.lastUsed.getTime())
+            .slice(0, 5);
+          
+          for (const pattern of patterns) {
+            timeline.push({
+              type: 'command',
+              timestamp: pattern.lastUsed,
+              description: pattern.description,
+              metadata: { pattern: pattern.pattern, category: pattern.category }
+            });
+          }
+        }
+
+        // Sort by timestamp
+        timeline.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+
+        res.json({ timeline: timeline.slice(0, 50) });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Context preview - what would be included in next request
+    this.app.post('/api/memory/context-preview', async (req, res) => {
+      try {
+        const { message } = req.body;
+        const context = await this.messageFlow.processMessage({
+          role: 'user',
+          content: message,
+          timestamp: new Date()
+        });
+
+        res.json({
+          context,
+          length: context.length,
+          sections: this.analyzeContextSections(context)
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+  }
+
+  private analyzeContextSections(context: string): any[] {
+    const sections = [];
+    const lines = context.split('\n');
+    let currentSection = null;
+    let currentContent = [];
+
+    for (const line of lines) {
+      if (line.startsWith('## ')) {
+        if (currentSection) {
+          sections.push({
+            title: currentSection,
+            lines: currentContent.length,
+            characters: currentContent.join('\n').length
+          });
+        }
+        currentSection = line.substring(3);
+        currentContent = [];
+      } else {
+        currentContent.push(line);
+      }
+    }
+
+    if (currentSection) {
+      sections.push({
+        title: currentSection,
+        lines: currentContent.length,
+        characters: currentContent.join('\n').length
+      });
+    }
+
+    return sections;
+  }
+
+  async start(port: number = 9486): Promise<void> {
+    return new Promise((resolve) => {
+      this.app.listen(port, () => {
+        this.emit('started', { port });
+        resolve();
+      });
+    });
+  }
+}

package/src/memory/index.ts

@@ -0,0 +1,187 @@
+/**
+ * Hivemind Memory System - Automatic context management for all agents
+ */
+
+import { MessageFlowIntegration, MessageFlowConfig } from './processors/message-flow-integration';
+import { ResearchDigester } from './processors/research-digester';
+import { CommandLearner } from './processors/command-learner';
+import { AgentSync } from './processors/agent-sync';
+import { DashboardIntegration } from './dashboard-integration';
+import { EventEmitter } from 'events';
+
+export interface MemorySystemConfig {
+  agentId: string;
+  agentName: string;
+  workspaceRoot: string;
+  memoryURL?: string;
+  context?: string;
+  maxContextTokens?: number;
+  dashboardPort?: number;
+  syncEndpoints?: Map<string, string>; // Other agents to sync with
+}
+
+export class MemorySystem extends EventEmitter {
+  private messageFlow: MessageFlowIntegration;
+  private researchDigester: ResearchDigester;
+  private commandLearner: CommandLearner;
+  private agentSync: AgentSync;
+  private dashboard?: DashboardIntegration;
+  private config: MemorySystemConfig;
+
+  constructor(config: MemorySystemConfig) {
+    super();
+    this.config = config;
+
+    // Initialize core message flow
+    this.messageFlow = new MessageFlowIntegration({
+      workspaceRoot: config.workspaceRoot,
+      memoryURL: config.memoryURL,
+      context: config.context,
+      maxContextTokens: config.maxContextTokens
+    });
+
+    // Initialize processors
+    this.researchDigester = new ResearchDigester();
+    this.commandLearner = new CommandLearner();
+    this.agentSync = new AgentSync(config.agentId, config.agentName);
+
+    // Register processors with message flow
+    this.messageFlow['processManager'].register('research-digester', this.researchDigester);
+    this.messageFlow['processManager'].register('command-learner', this.commandLearner);
+    this.messageFlow['processManager'].register('agent-sync', this.agentSync);
+
+    // Set up event forwarding
+    this.setupEventForwarding();
+
+    // Register sync endpoints if provided
+    if (config.syncEndpoints) {
+      for (const [agentId, endpoint] of config.syncEndpoints) {
+        this.agentSync.registerAgent(agentId, agentId, endpoint);
+      }
+    }
+  }
+
+  private setupEventForwarding(): void {
+    // Forward errors
+    this.messageFlow.on('error', (error) => this.emit('error', error));
+    this.researchDigester.on('error', (error) => this.emit('error', { source: 'research', error }));
+    this.commandLearner.on('error', (error) => this.emit('error', { source: 'commands', error }));
+    this.agentSync.on('error', (error) => this.emit('error', { source: 'sync', error }));
+
+    // Forward interesting events
+    this.messageFlow.on('context-built', (data) => this.emit('context-built', data));
+    this.researchDigester.on('research-digested', (entry) => this.emit('research-added', entry));
+    this.commandLearner.on('patterns-updated', (stats) => this.emit('commands-updated', stats));
+    this.agentSync.on('handoff-received', (handoff) => this.emit('task-handoff', handoff));
+  }
+
+  /**
+   * Start the memory system
+   */
+  async start(): Promise<void> {
+    // Start message flow (which starts all processors)
+    await this.messageFlow.start();
+
+    // Start dashboard if port specified
+    if (this.config.dashboardPort) {
+      this.dashboard = new DashboardIntegration({
+        port: this.config.dashboardPort,
+        messageFlow: this.messageFlow,
+        researchDigester: this.researchDigester,
+        commandLearner: this.commandLearner,
+        agentSync: this.agentSync
+      });
+
+      await this.dashboard.start(this.config.dashboardPort);
+      this.emit('dashboard-started', { port: this.config.dashboardPort });
+    }
+
+    this.emit('started');
+  }
+
+  /**
+   * Stop the memory system
+   */
+  async stop(): Promise<void> {
+    await this.messageFlow.stop();
+    this.emit('stopped');
+  }
+
+  /**
+   * Process a message and get enriched context
+   */
+  async processMessage(message: {
+    role: 'user' | 'assistant';
+    content: string;
+    metadata?: Record<string, any>;
+  }): Promise<string> {
+    return this.messageFlow.processMessage({
+      ...message,
+      timestamp: new Date(),
+      context: this.config.context
+    });
+  }
+
+  /**
+   * Add research content (from web pages, PDFs, etc.)
+   */
+  async addResearch(content: string, metadata?: {
+    url?: string;
+    title?: string;
+    sourceType?: 'web' | 'pdf' | 'markdown' | 'other';
+  }): Promise<void> {
+    await this.researchDigester.addContent(content, metadata);
+  }
+
+  /**
+   * Track a command execution
+   */
+  async trackCommand(command: string, context: string, output?: string, success: boolean = true): Promise<void> {
+    await this.commandLearner.trackCommand(command, context, output, success);
+  }
+
+  /**
+   * Share a task with other agents
+   */
+  async shareTask(task: {
+    description: string;
+    status: 'active' | 'completed' | 'blocked';
+    dependencies?: string[];
+    progress?: number;
+    notes?: string[];
+  }): Promise<string> {
+    return this.agentSync.shareTask({
+      ...task,
+      assignedTo: this.config.agentName,
+      dependencies: task.dependencies || [],
+      progress: task.progress || 0,
+      notes: task.notes || []
+    });
+  }
+
+  /**
+   * Share an insight
+   */
+  async shareInsight(insight: string, context?: string): Promise<void> {
+    await this.agentSync.shareInsight(insight, context);
+  }
+
+  /**
+   * Get current system state
+   */
+  async getState(): Promise<any> {
+    return this.messageFlow.getState();
+  }
+}
+
+// Export all components for advanced usage
+export { MessageFlowIntegration } from './processors/message-flow-integration';
+export { BackgroundProcessor, ProcessManager } from './processors/background-processor';
+export { CodeIndexer } from './processors/code-indexer';
+export { TaskTracker } from './processors/task-tracker';
+export { ResearchDigester } from './processors/research-digester';
+export { CommandLearner } from './processors/command-learner';
+export { AgentSync } from './processors/agent-sync';
+export { ContextManager } from './processors/context-manager';
+export { MemoryAPIClient } from './processors/memory-api-client';
+export { DashboardIntegration } from './dashboard-integration';

package/src/memory/performance-test.ts

@@ -0,0 +1,208 @@
+/**
+ * Performance test for the memory management system
+ */
+
+import { MemorySystem } from './index';
+import * as fs from 'fs';
+import * as path from 'path';
+
+async function runPerformanceTest() {
+  console.log('🧪 Starting Hivemind Memory System Performance Test\n');
+
+  const startTime = Date.now();
+  const workspaceRoot = path.join(__dirname, '../../test-workspace');
+
+  // Create test workspace
+  if (!fs.existsSync(workspaceRoot)) {
+    fs.mkdirSync(workspaceRoot, { recursive: true });
+  }
+
+  // Initialize memory system
+  const memory = new MemorySystem({
+    agentId: 'test-agent-001',
+    agentName: 'TestAgent',
+    workspaceRoot,
+    memoryURL: 'http://localhost:3434',
+    context: 'performance-test',
+    maxContextTokens: 8000,
+    dashboardPort: 9487
+  });
+
+  // Track metrics
+  const metrics = {
+    messagesProcessed: 0,
+    contextBuildTime: [] as number[],
+    memoryUsage: [] as number[],
+    errors: 0
+  };
+
+  // Set up event listeners
+  memory.on('error', (error) => {
+    console.error('❌ Error:', error);
+    metrics.errors++;
+  });
+
+  memory.on('context-built', (data) => {
+    console.log(`📝 Context built: ${data.contextLength} chars, ${data.sections} sections`);
+  });
+
+  memory.on('started', () => {
+    console.log('✅ Memory system started\n');
+  });
+
+  memory.on('dashboard-started', ({ port }) => {
+    console.log(`🌐 Dashboard available at http://localhost:${port}\n`);
+  });
+
+  try {
+    // Start the system
+    await memory.start();
+
+    // Test 1: Message processing
+    console.log('📊 Test 1: Message Processing');
+    const messages = [
+      { role: 'user' as const, content: 'Can you help me debug the authentication system in src/auth/login.ts?' },
+      { role: 'assistant' as const, content: 'I\'ll help you debug the authentication system. Let me look at src/auth/login.ts...' },
+      { role: 'user' as const, content: 'The error happens when users try to login with OAuth' },
+      { role: 'assistant' as const, content: 'I see. Let me check the OAuth configuration in src/auth/oauth.ts and src/config/oauth.json' }
+    ];
+
+    for (const message of messages) {
+      const startContext = Date.now();
+      const context = await memory.processMessage(message);
+      const contextTime = Date.now() - startContext;
+      
+      metrics.contextBuildTime.push(contextTime);
+      metrics.messagesProcessed++;
+      
+      console.log(`  - Processed ${message.role} message in ${contextTime}ms (context: ${context.length} chars)`);
+    }
+
+    // Test 2: Research ingestion
+    console.log('\n📊 Test 2: Research Ingestion');
+    const researchContent = `
+# Understanding TypeScript Decorators
+
+Decorators are a powerful feature in TypeScript that allow you to modify classes and their members.
+
+## Key Points
+- Decorators use the @ symbol
+- They can be applied to classes, methods, properties, and parameters
+- Decorators are executed at runtime
+- Multiple decorators can be composed
+
+## Common Use Cases
+- Logging and monitoring
+- Validation
+- Dependency injection
+- Route handling in web frameworks
+
+## Example
+\`\`\`typescript
+@Injectable()
+class UserService {
+  @Log()
+  async getUser(id: string) {
+    return this.db.findUser(id);
+  }
+}
+\`\`\`
+`;
+
+    await memory.addResearch(researchContent, {
+      title: 'TypeScript Decorators Guide',
+      sourceType: 'markdown'
+    });
+    console.log('  - Added research document');
+
+    // Test 3: Command tracking
+    console.log('\n📊 Test 3: Command Tracking');
+    const commands = [
+      { cmd: 'git status', output: 'On branch main\nnothing to commit', success: true },
+      { cmd: 'git pull origin main', output: 'Already up to date.', success: true },
+      { cmd: 'npm test', output: 'Test suite failed', success: false },
+      { cmd: 'npm test -- --fix', output: 'All tests passed', success: true }
+    ];
+
+    for (const { cmd, output, success } of commands) {
+      await memory.trackCommand(cmd, 'debugging auth system', output, success);
+      console.log(`  - Tracked command: ${cmd} (${success ? '✓' : '✗'})`);
+    }
+
+    // Test 4: Task sharing
+    console.log('\n📊 Test 4: Task Management');
+    const taskId = await memory.shareTask({
+      description: 'Fix OAuth authentication bug',
+      status: 'active',
+      progress: 45,
+      notes: ['Users report 401 errors', 'Might be related to token expiration']
+    });
+    console.log(`  - Created shared task: ${taskId}`);
+
+    // Test 5: Insight sharing
+    await memory.shareInsight('OAuth tokens need refresh logic to handle expiration gracefully');
+    console.log('  - Shared insight about OAuth tokens');
+
+    // Test 6: Memory usage
+    console.log('\n📊 Test 6: Memory Usage');
+    for (let i = 0; i < 5; i++) {
+      const usage = process.memoryUsage();
+      metrics.memoryUsage.push(usage.heapUsed);
+      console.log(`  - Heap used: ${(usage.heapUsed / 1024 / 1024).toFixed(2)} MB`);
+      
+      // Process more messages to see memory growth
+      await memory.processMessage({
+        role: 'user',
+        content: `Test message ${i}: Working on feature ${i}`
+      });
+      
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+
+    // Test 7: Context building performance with large content
+    console.log('\n📊 Test 7: Large Context Performance');
+    const largeMessage = 'x'.repeat(1000) + ' Can you analyze this large codebase?';
+    const largeStart = Date.now();
+    const largeContext = await memory.processMessage({
+      role: 'user',
+      content: largeMessage
+    });
+    const largeTime = Date.now() - largeStart;
+    console.log(`  - Large message processed in ${largeTime}ms (context: ${largeContext.length} chars)`);
+
+    // Get final state
+    const state = await memory.getState();
+    console.log('\n📊 Final System State:');
+    console.log(`  - Memory episodes: ${state.memory?.episodes || 0}`);
+    console.log(`  - Active tasks: ${state.tasks?.length || 0}`);
+    console.log(`  - Working set files: ${state.workingSet?.length || 0}`);
+    console.log(`  - Background processors: ${state.processors || 0}`);
+
+    // Calculate statistics
+    const avgContextTime = metrics.contextBuildTime.reduce((a, b) => a + b, 0) / metrics.contextBuildTime.length;
+    const maxMemory = Math.max(...metrics.memoryUsage);
+    const minMemory = Math.min(...metrics.memoryUsage);
+
+    console.log('\n📈 Performance Summary:');
+    console.log(`  - Total runtime: ${((Date.now() - startTime) / 1000).toFixed(2)}s`);
+    console.log(`  - Messages processed: ${metrics.messagesProcessed}`);
+    console.log(`  - Avg context build time: ${avgContextTime.toFixed(2)}ms`);
+    console.log(`  - Memory usage: ${(minMemory / 1024 / 1024).toFixed(2)} - ${(maxMemory / 1024 / 1024).toFixed(2)} MB`);
+    console.log(`  - Errors encountered: ${metrics.errors}`);
+
+    // Stop the system
+    await memory.stop();
+    console.log('\n✅ Memory system stopped successfully');
+
+  } catch (error) {
+    console.error('\n❌ Test failed:', error);
+    process.exit(1);
+  }
+
+  // Clean up test workspace
+  fs.rmSync(workspaceRoot, { recursive: true, force: true });
+  console.log('\n🧹 Cleaned up test workspace');
+}
+
+// Run the test
+runPerformanceTest().catch(console.error);