@sesamespace/hivemind 0.8.10 → 0.8.11

package/src/memory/index.ts

@@ -0,0 +1,187 @@
+/**
+ * Hivemind Memory System - Automatic context management for all agents
+ */
+
+import { MessageFlowIntegration, MessageFlowConfig } from './processors/message-flow-integration';
+import { ResearchDigester } from './processors/research-digester';
+import { CommandLearner } from './processors/command-learner';
+import { AgentSync } from './processors/agent-sync';
+import { DashboardIntegration } from './dashboard-integration';
+import { EventEmitter } from 'events';
+
+export interface MemorySystemConfig {
+  agentId: string;
+  agentName: string;
+  workspaceRoot: string;
+  memoryURL?: string;
+  context?: string;
+  maxContextTokens?: number;
+  dashboardPort?: number;
+  syncEndpoints?: Map<string, string>; // Other agents to sync with
+}
+
+export class MemorySystem extends EventEmitter {
+  private messageFlow: MessageFlowIntegration;
+  private researchDigester: ResearchDigester;
+  private commandLearner: CommandLearner;
+  private agentSync: AgentSync;
+  private dashboard?: DashboardIntegration;
+  private config: MemorySystemConfig;
+
+  constructor(config: MemorySystemConfig) {
+    super();
+    this.config = config;
+
+    // Initialize core message flow
+    this.messageFlow = new MessageFlowIntegration({
+      workspaceRoot: config.workspaceRoot,
+      memoryURL: config.memoryURL,
+      context: config.context,
+      maxContextTokens: config.maxContextTokens
+    });
+
+    // Initialize processors
+    this.researchDigester = new ResearchDigester();
+    this.commandLearner = new CommandLearner();
+    this.agentSync = new AgentSync(config.agentId, config.agentName);
+
+    // Register processors with message flow
+    this.messageFlow['processManager'].register('research-digester', this.researchDigester);
+    this.messageFlow['processManager'].register('command-learner', this.commandLearner);
+    this.messageFlow['processManager'].register('agent-sync', this.agentSync);
+
+    // Set up event forwarding
+    this.setupEventForwarding();
+
+    // Register sync endpoints if provided
+    if (config.syncEndpoints) {
+      for (const [agentId, endpoint] of config.syncEndpoints) {
+        this.agentSync.registerAgent(agentId, agentId, endpoint);
+      }
+    }
+  }
+
+  private setupEventForwarding(): void {
+    // Forward errors
+    this.messageFlow.on('error', (error) => this.emit('error', error));
+    this.researchDigester.on('error', (error) => this.emit('error', { source: 'research', error }));
+    this.commandLearner.on('error', (error) => this.emit('error', { source: 'commands', error }));
+    this.agentSync.on('error', (error) => this.emit('error', { source: 'sync', error }));
+
+    // Forward interesting events
+    this.messageFlow.on('context-built', (data) => this.emit('context-built', data));
+    this.researchDigester.on('research-digested', (entry) => this.emit('research-added', entry));
+    this.commandLearner.on('patterns-updated', (stats) => this.emit('commands-updated', stats));
+    this.agentSync.on('handoff-received', (handoff) => this.emit('task-handoff', handoff));
+  }
+
+  /**
+   * Start the memory system
+   */
+  async start(): Promise<void> {
+    // Start message flow (which starts all processors)
+    await this.messageFlow.start();
+
+    // Start dashboard if port specified
+    if (this.config.dashboardPort) {
+      this.dashboard = new DashboardIntegration({
+        port: this.config.dashboardPort,
+        messageFlow: this.messageFlow,
+        researchDigester: this.researchDigester,
+        commandLearner: this.commandLearner,
+        agentSync: this.agentSync
+      });
+
+      await this.dashboard.start(this.config.dashboardPort);
+      this.emit('dashboard-started', { port: this.config.dashboardPort });
+    }
+
+    this.emit('started');
+  }
+
+  /**
+   * Stop the memory system
+   */
+  async stop(): Promise<void> {
+    await this.messageFlow.stop();
+    this.emit('stopped');
+  }
+
+  /**
+   * Process a message and get enriched context
+   */
+  async processMessage(message: {
+    role: 'user' | 'assistant';
+    content: string;
+    metadata?: Record<string, any>;
+  }): Promise<string> {
+    return this.messageFlow.processMessage({
+      ...message,
+      timestamp: new Date(),
+      context: this.config.context
+    });
+  }
+
+  /**
+   * Add research content (from web pages, PDFs, etc.)
+   */
+  async addResearch(content: string, metadata?: {
+    url?: string;
+    title?: string;
+    sourceType?: 'web' | 'pdf' | 'markdown' | 'other';
+  }): Promise<void> {
+    await this.researchDigester.addContent(content, metadata);
+  }
+
+  /**
+   * Track a command execution
+   */
+  async trackCommand(command: string, context: string, output?: string, success: boolean = true): Promise<void> {
+    await this.commandLearner.trackCommand(command, context, output, success);
+  }
+
+  /**
+   * Share a task with other agents
+   */
+  async shareTask(task: {
+    description: string;
+    status: 'active' | 'completed' | 'blocked';
+    dependencies?: string[];
+    progress?: number;
+    notes?: string[];
+  }): Promise<string> {
+    return this.agentSync.shareTask({
+      ...task,
+      assignedTo: this.config.agentName,
+      dependencies: task.dependencies || [],
+      progress: task.progress || 0,
+      notes: task.notes || []
+    });
+  }
+
+  /**
+   * Share an insight
+   */
+  async shareInsight(insight: string, context?: string): Promise<void> {
+    await this.agentSync.shareInsight(insight, context);
+  }
+
+  /**
+   * Get current system state
+   */
+  async getState(): Promise<any> {
+    return this.messageFlow.getState();
+  }
+}
+
+// Export all components for advanced usage
+export { MessageFlowIntegration } from './processors/message-flow-integration';
+export { BackgroundProcessor, ProcessManager } from './processors/background-processor';
+export { CodeIndexer } from './processors/code-indexer';
+export { TaskTracker } from './processors/task-tracker';
+export { ResearchDigester } from './processors/research-digester';
+export { CommandLearner } from './processors/command-learner';
+export { AgentSync } from './processors/agent-sync';
+export { ContextManager } from './processors/context-manager';
+export { MemoryAPIClient } from './processors/memory-api-client';
+export { DashboardIntegration } from './dashboard-integration';

package/src/memory/performance-test.ts

@@ -0,0 +1,208 @@
+/**
+ * Performance test for the memory management system
+ */
+
+import { MemorySystem } from './index';
+import * as fs from 'fs';
+import * as path from 'path';
+
+async function runPerformanceTest() {
+  console.log('🧪 Starting Hivemind Memory System Performance Test\n');
+
+  const startTime = Date.now();
+  const workspaceRoot = path.join(__dirname, '../../test-workspace');
+
+  // Create test workspace
+  if (!fs.existsSync(workspaceRoot)) {
+    fs.mkdirSync(workspaceRoot, { recursive: true });
+  }
+
+  // Initialize memory system
+  const memory = new MemorySystem({
+    agentId: 'test-agent-001',
+    agentName: 'TestAgent',
+    workspaceRoot,
+    memoryURL: 'http://localhost:3434',
+    context: 'performance-test',
+    maxContextTokens: 8000,
+    dashboardPort: 9487
+  });
+
+  // Track metrics
+  const metrics = {
+    messagesProcessed: 0,
+    contextBuildTime: [] as number[],
+    memoryUsage: [] as number[],
+    errors: 0
+  };
+
+  // Set up event listeners
+  memory.on('error', (error) => {
+    console.error('❌ Error:', error);
+    metrics.errors++;
+  });
+
+  memory.on('context-built', (data) => {
+    console.log(`📝 Context built: ${data.contextLength} chars, ${data.sections} sections`);
+  });
+
+  memory.on('started', () => {
+    console.log('✅ Memory system started\n');
+  });
+
+  memory.on('dashboard-started', ({ port }) => {
+    console.log(`🌐 Dashboard available at http://localhost:${port}\n`);
+  });
+
+  try {
+    // Start the system
+    await memory.start();
+
+    // Test 1: Message processing
+    console.log('📊 Test 1: Message Processing');
+    const messages = [
+      { role: 'user' as const, content: 'Can you help me debug the authentication system in src/auth/login.ts?' },
+      { role: 'assistant' as const, content: 'I\'ll help you debug the authentication system. Let me look at src/auth/login.ts...' },
+      { role: 'user' as const, content: 'The error happens when users try to login with OAuth' },
+      { role: 'assistant' as const, content: 'I see. Let me check the OAuth configuration in src/auth/oauth.ts and src/config/oauth.json' }
+    ];
+
+    for (const message of messages) {
+      const startContext = Date.now();
+      const context = await memory.processMessage(message);
+      const contextTime = Date.now() - startContext;
+      
+      metrics.contextBuildTime.push(contextTime);
+      metrics.messagesProcessed++;
+      
+      console.log(`  - Processed ${message.role} message in ${contextTime}ms (context: ${context.length} chars)`);
+    }
+
+    // Test 2: Research ingestion
+    console.log('\n📊 Test 2: Research Ingestion');
+    const researchContent = `
+# Understanding TypeScript Decorators
+
+Decorators are a powerful feature in TypeScript that allow you to modify classes and their members.
+
+## Key Points
+- Decorators use the @ symbol
+- They can be applied to classes, methods, properties, and parameters
+- Decorators are executed at runtime
+- Multiple decorators can be composed
+
+## Common Use Cases
+- Logging and monitoring
+- Validation
+- Dependency injection
+- Route handling in web frameworks
+
+## Example
+\`\`\`typescript
+@Injectable()
+class UserService {
+  @Log()
+  async getUser(id: string) {
+    return this.db.findUser(id);
+  }
+}
+\`\`\`
+`;
+
+    await memory.addResearch(researchContent, {
+      title: 'TypeScript Decorators Guide',
+      sourceType: 'markdown'
+    });
+    console.log('  - Added research document');
+
+    // Test 3: Command tracking
+    console.log('\n📊 Test 3: Command Tracking');
+    const commands = [
+      { cmd: 'git status', output: 'On branch main\nnothing to commit', success: true },
+      { cmd: 'git pull origin main', output: 'Already up to date.', success: true },
+      { cmd: 'npm test', output: 'Test suite failed', success: false },
+      { cmd: 'npm test -- --fix', output: 'All tests passed', success: true }
+    ];
+
+    for (const { cmd, output, success } of commands) {
+      await memory.trackCommand(cmd, 'debugging auth system', output, success);
+      console.log(`  - Tracked command: ${cmd} (${success ? '✓' : '✗'})`);
+    }
+
+    // Test 4: Task sharing
+    console.log('\n📊 Test 4: Task Management');
+    const taskId = await memory.shareTask({
+      description: 'Fix OAuth authentication bug',
+      status: 'active',
+      progress: 45,
+      notes: ['Users report 401 errors', 'Might be related to token expiration']
+    });
+    console.log(`  - Created shared task: ${taskId}`);
+
+    // Test 5: Insight sharing
+    await memory.shareInsight('OAuth tokens need refresh logic to handle expiration gracefully');
+    console.log('  - Shared insight about OAuth tokens');
+
+    // Test 6: Memory usage
+    console.log('\n📊 Test 6: Memory Usage');
+    for (let i = 0; i < 5; i++) {
+      const usage = process.memoryUsage();
+      metrics.memoryUsage.push(usage.heapUsed);
+      console.log(`  - Heap used: ${(usage.heapUsed / 1024 / 1024).toFixed(2)} MB`);
+      
+      // Process more messages to see memory growth
+      await memory.processMessage({
+        role: 'user',
+        content: `Test message ${i}: Working on feature ${i}`
+      });
+      
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+
+    // Test 7: Context building performance with large content
+    console.log('\n📊 Test 7: Large Context Performance');
+    const largeMessage = 'x'.repeat(1000) + ' Can you analyze this large codebase?';
+    const largeStart = Date.now();
+    const largeContext = await memory.processMessage({
+      role: 'user',
+      content: largeMessage
+    });
+    const largeTime = Date.now() - largeStart;
+    console.log(`  - Large message processed in ${largeTime}ms (context: ${largeContext.length} chars)`);
+
+    // Get final state
+    const state = await memory.getState();
+    console.log('\n📊 Final System State:');
+    console.log(`  - Memory episodes: ${state.memory?.episodes || 0}`);
+    console.log(`  - Active tasks: ${state.tasks?.length || 0}`);
+    console.log(`  - Working set files: ${state.workingSet?.length || 0}`);
+    console.log(`  - Background processors: ${state.processors || 0}`);
+
+    // Calculate statistics
+    const avgContextTime = metrics.contextBuildTime.reduce((a, b) => a + b, 0) / metrics.contextBuildTime.length;
+    const maxMemory = Math.max(...metrics.memoryUsage);
+    const minMemory = Math.min(...metrics.memoryUsage);
+
+    console.log('\n📈 Performance Summary:');
+    console.log(`  - Total runtime: ${((Date.now() - startTime) / 1000).toFixed(2)}s`);
+    console.log(`  - Messages processed: ${metrics.messagesProcessed}`);
+    console.log(`  - Avg context build time: ${avgContextTime.toFixed(2)}ms`);
+    console.log(`  - Memory usage: ${(minMemory / 1024 / 1024).toFixed(2)} - ${(maxMemory / 1024 / 1024).toFixed(2)} MB`);
+    console.log(`  - Errors encountered: ${metrics.errors}`);
+
+    // Stop the system
+    await memory.stop();
+    console.log('\n✅ Memory system stopped successfully');
+
+  } catch (error) {
+    console.error('\n❌ Test failed:', error);
+    process.exit(1);
+  }
+
+  // Clean up test workspace
+  fs.rmSync(workspaceRoot, { recursive: true, force: true });
+  console.log('\n🧹 Cleaned up test workspace');
+}
+
+// Run the test
+runPerformanceTest().catch(console.error);

package/src/memory/processors/agent-sync.ts

@@ -0,0 +1,312 @@
+/**
+ * Agent Sync - Shares knowledge between multiple Hivemind agents
+ */
+
+import { BackgroundProcessor } from './background-processor';
+import { EventEmitter } from 'events';
+import axios from 'axios';
+
+export interface AgentKnowledge {
+  agentId: string;
+  agentName: string;
+  context: string;
+  tasks: SharedTask[];
+  insights: string[];
+  capabilities: string[];
+  lastSync: Date;
+}
+
+export interface SharedTask {
+  id: string;
+  description: string;
+  status: 'active' | 'completed' | 'blocked';
+  assignedTo: string;
+  dependencies: string[];
+  progress: number;
+  notes: string[];
+}
+
+export interface SyncMessage {
+  type: 'task-update' | 'insight' | 'handoff' | 'capability';
+  fromAgent: string;
+  toAgent?: string; // Optional for broadcasts
+  payload: any;
+  timestamp: Date;
+}
+
+export class AgentSync extends BackgroundProcessor {
+  private agentId: string;
+  private agentName: string;
+  private otherAgents: Map<string, AgentKnowledge> = new Map();
+  private syncQueue: SyncMessage[] = [];
+  private sharedTasks: Map<string, SharedTask> = new Map();
+  private syncEndpoints: Map<string, string> = new Map(); // agentId -> endpoint
+
+  constructor(agentId: string, agentName: string) {
+    super('agent-sync', 30000); // Sync every 30 seconds
+    this.agentId = agentId;
+    this.agentName = agentName;
+  }
+
+  async process(): Promise<void> {
+    // Process outgoing sync messages
+    await this.processSyncQueue();
+
+    // Pull updates from other agents
+    await this.pullUpdates();
+
+    // Check for stale tasks that might need handoff
+    await this.checkHandoffs();
+
+    this.emit('sync-complete', {
+      agentsKnown: this.otherAgents.size,
+      sharedTasks: this.sharedTasks.size,
+      queuedMessages: this.syncQueue.length
+    });
+  }
+
+  /**
+   * Register another agent for syncing
+   */
+  async registerAgent(agentId: string, agentName: string, endpoint: string): Promise<void> {
+    this.syncEndpoints.set(agentId, endpoint);
+    this.otherAgents.set(agentId, {
+      agentId,
+      agentName,
+      context: 'unknown',
+      tasks: [],
+      insights: [],
+      capabilities: [],
+      lastSync: new Date()
+    });
+  }
+
+  /**
+   * Share a task with other agents
+   */
+  async shareTask(task: Omit<SharedTask, 'id'>): Promise<string> {
+    const taskId = `${this.agentId}-${Date.now()}`;
+    const sharedTask: SharedTask = {
+      id: taskId,
+      ...task
+    };
+
+    this.sharedTasks.set(taskId, sharedTask);
+    
+    // Broadcast task to other agents
+    this.queueSync({
+      type: 'task-update',
+      fromAgent: this.agentId,
+      payload: sharedTask,
+      timestamp: new Date()
+    });
+
+    return taskId;
+  }
+
+  /**
+   * Share an insight learned during work
+   */
+  async shareInsight(insight: string, context?: string): Promise<void> {
+    this.queueSync({
+      type: 'insight',
+      fromAgent: this.agentId,
+      payload: { insight, context },
+      timestamp: new Date()
+    });
+  }
+
+  /**
+   * Announce a new capability
+   */
+  async announceCapability(capability: string): Promise<void> {
+    this.queueSync({
+      type: 'capability',
+      fromAgent: this.agentId,
+      payload: { capability },
+      timestamp: new Date()
+    });
+  }
+
+  /**
+   * Request handoff of a task to another agent
+   */
+  async requestHandoff(taskId: string, toAgent: string, reason: string): Promise<void> {
+    const task = this.sharedTasks.get(taskId);
+    if (!task) return;
+
+    this.queueSync({
+      type: 'handoff',
+      fromAgent: this.agentId,
+      toAgent,
+      payload: { taskId, task, reason },
+      timestamp: new Date()
+    });
+  }
+
+  private queueSync(message: SyncMessage): void {
+    this.syncQueue.push(message);
+  }
+
+  private async processSyncQueue(): Promise<void> {
+    while (this.syncQueue.length > 0) {
+      const message = this.syncQueue.shift();
+      if (!message) continue;
+
+      if (message.toAgent) {
+        // Direct message to specific agent
+        await this.sendToAgent(message.toAgent, message);
+      } else {
+        // Broadcast to all agents
+        for (const agentId of this.syncEndpoints.keys()) {
+          if (agentId !== this.agentId) {
+            await this.sendToAgent(agentId, message);
+          }
+        }
+      }
+    }
+  }
+
+  private async sendToAgent(agentId: string, message: SyncMessage): Promise<void> {
+    const endpoint = this.syncEndpoints.get(agentId);
+    if (!endpoint) return;
+
+    try {
+      await axios.post(`${endpoint}/sync`, message, {
+        timeout: 5000,
+        headers: {
+          'X-Agent-ID': this.agentId,
+          'X-Agent-Name': this.agentName
+        }
+      });
+    } catch (error) {
+      this.emit('sync-error', { agentId, error });
+    }
+  }
+
+  private async pullUpdates(): Promise<void> {
+    for (const [agentId, endpoint] of this.syncEndpoints) {
+      if (agentId === this.agentId) continue;
+
+      try {
+        const response = await axios.get(`${endpoint}/sync/updates`, {
+          params: {
+            since: this.otherAgents.get(agentId)?.lastSync.toISOString()
+          },
+          timeout: 5000,
+          headers: {
+            'X-Agent-ID': this.agentId
+          }
+        });
+
+        if (response.data.updates) {
+          await this.processUpdates(agentId, response.data.updates);
+        }
+      } catch (error) {
+        this.emit('pull-error', { agentId, error });
+      }
+    }
+  }
+
+  private async processUpdates(fromAgent: string, updates: SyncMessage[]): Promise<void> {
+    const agentKnowledge = this.otherAgents.get(fromAgent);
+    if (!agentKnowledge) return;
+
+    for (const update of updates) {
+      switch (update.type) {
+        case 'task-update':
+          const task = update.payload as SharedTask;
+          this.sharedTasks.set(task.id, task);
+          agentKnowledge.tasks = agentKnowledge.tasks.filter(t => t.id !== task.id);
+          agentKnowledge.tasks.push(task);
+          break;
+
+        case 'insight':
+          const { insight } = update.payload;
+          if (!agentKnowledge.insights.includes(insight)) {
+            agentKnowledge.insights.push(insight);
+          }
+          break;
+
+        case 'capability':
+          const { capability } = update.payload;
+          if (!agentKnowledge.capabilities.includes(capability)) {
+            agentKnowledge.capabilities.push(capability);
+          }
+          break;
+
+        case 'handoff':
+          if (update.toAgent === this.agentId) {
+            await this.handleHandoff(update.payload);
+          }
+          break;
+      }
+    }
+
+    agentKnowledge.lastSync = new Date();
+  }
+
+  private async handleHandoff(payload: any): Promise<void> {
+    const { taskId, task, reason } = payload;
+    
+    // Accept the handoff
+    task.assignedTo = this.agentName;
+    this.sharedTasks.set(taskId, task);
+
+    this.emit('handoff-received', { taskId, task, reason });
+
+    // Acknowledge the handoff
+    this.queueSync({
+      type: 'task-update',
+      fromAgent: this.agentId,
+      payload: task,
+      timestamp: new Date()
+    });
+  }
+
+  private async checkHandoffs(): Promise<void> {
+    // Check for tasks that might need handoff
+    for (const task of this.sharedTasks.values()) {
+      if (task.assignedTo === this.agentName && task.status === 'blocked') {
+        // Check if another agent might be better suited
+        const betterAgent = this.findBetterAgent(task);
+        if (betterAgent) {
+          await this.requestHandoff(
+            task.id,
+            betterAgent,
+            'Task blocked, another agent may have required capabilities'
+          );
+        }
+      }
+    }
+  }
+
+  private findBetterAgent(task: SharedTask): string | null {
+    // Simple heuristic: find agent with relevant capabilities
+    for (const [agentId, knowledge] of this.otherAgents) {
+      // Check if agent has capabilities mentioned in task notes
+      for (const note of task.notes) {
+        for (const capability of knowledge.capabilities) {
+          if (note.toLowerCase().includes(capability.toLowerCase())) {
+            return agentId;
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Get knowledge about other agents
+   */
+  async getAgentKnowledge(): Promise<AgentKnowledge[]> {
+    return Array.from(this.otherAgents.values());
+  }
+
+  /**
+   * Get all shared tasks
+   */
+  async getSharedTasks(): Promise<SharedTask[]> {
+    return Array.from(this.sharedTasks.values());
+  }
+}