@sesamespace/hivemind 0.8.10 → 0.8.11

package/MEMORY-ENHANCEMENT-PLAN.md

@@ -0,0 +1,211 @@
+# Memory Enhancement Plan — Automatic Context Management
+
+## Vision
+Transform Hivemind's memory system from passive storage to active context management. Background processes continuously organize, index, and surface relevant information without agent intervention.
+
+## Core Enhancements
+
+### 1. Code Context Tracking
+**Background Process:** `code-indexer`
+- Monitors file access patterns (which files the agent reads/writes)
+- Extracts key structures: functions, classes, interfaces, schemas
+- Maintains a "working set" of active code elements
+- Updates git commit context automatically
+- Indexes by: project, language, purpose, last-accessed
+
+**Data Structure:**
+```json
+{
+  "type": "code_context",
+  "project": "hivemind",
+  "file": "packages/runtime/src/agent.ts",
+  "elements": [
+    {
+      "name": "processMessage",
+      "type": "function",
+      "signature": "(msg: Message): Promise<Response>",
+      "purpose": "Core message processing loop",
+      "dependencies": ["memory-client", "router"]
+    }
+  ],
+  "last_accessed": "2024-01-15T10:30:00Z",
+  "access_count": 15,
+  "git_context": {
+    "branch": "feature/memory-enhancement",
+    "last_commit": "abc123",
+    "modified": true
+  }
+}
+```
+
+### 2. Web Research Digestion
+**Background Process:** `research-digester`
+- Monitors web fetch/browse operations
+- Extracts key concepts, APIs, solutions
+- Links research to active tasks/projects
+- Builds knowledge graph of related concepts
+- Identifies patterns across multiple sources
+
+**Data Structure:**
+```json
+{
+  "type": "research_insight",
+  "url": "https://docs.example.com/api",
+  "project": "sesame-integration",
+  "extracted": {
+    "key_concepts": ["OAuth flow", "webhook endpoints"],
+    "code_examples": ["const auth = await getToken()..."],
+    "warnings": ["Rate limit: 100 req/min"],
+    "related_to": ["auth-implementation", "rate-limiting"]
+  },
+  "timestamp": "2024-01-15T09:00:00Z",
+  "referenced_count": 3
+}
+```
+
+### 3. Task State Management
+**Background Process:** `task-tracker`
+- Monitors agent actions and maps to task progress
+- Detects task transitions (started, blocked, completed)
+- Tracks dependencies and blockers
+- Identifies patterns in task completion
+- Surfaces relevant context when returning to a task
+
+**Data Structure:**
+```json
+{
+  "type": "task_state",
+  "id": "implement-dashboard",
+  "project": "hivemind",
+  "status": "in_progress",
+  "progress": {
+    "completed": ["setup routes", "basic UI"],
+    "current": "implement request filtering",
+    "next": ["add export functionality", "write tests"]
+  },
+  "context": {
+    "key_files": ["src/dashboard/server.js", "src/dashboard/index.html"],
+    "recent_decisions": ["use server-sent events for real-time updates"],
+    "blockers": [],
+    "time_spent": "3.5 hours"
+  },
+  "last_updated": "2024-01-15T11:00:00Z"
+}
+```
+
+### 4. Tool Usage Patterns
+**Background Process:** `command-learner`
+- Tracks successful command sequences
+- Identifies common patterns and workflows
+- Builds "recipes" for common tasks
+- Learns from failures and corrections
+- Suggests optimizations
+
+**Data Structure:**
+```json
+{
+  "type": "tool_pattern",
+  "name": "github-push-workflow",
+  "triggers": ["git push", "push changes"],
+  "sequence": [
+    {"tool": "git_status", "check": "has_changes"},
+    {"tool": "git_add", "params": {"files": "."}}
+    {"tool": "git_commit", "params": {"message": "<generated>"}},
+    {"tool": "git_push", "params": {"remote": "origin"}}
+  ],
+  "success_rate": 0.95,
+  "last_used": "2024-01-15T10:00:00Z",
+  "variations": ["with-specific-files", "force-push"]
+}
+```
+
+### 5. Cross-Agent Awareness
+**Background Process:** `agent-sync`
+- Monitors Sesame channels for other agent activity
+- Extracts "public knowledge" from agent interactions
+- Tracks handoff points and collaboration patterns
+- Maintains agent capability registry
+- Identifies complementary skills
+
+**Data Structure:**
+```json
+{
+  "type": "agent_knowledge",
+  "agent": "bailey",
+  "capabilities": ["rust development", "system architecture"],
+  "current_focus": ["hivemind dashboard", "memory optimization"],
+  "collaboration_points": [
+    {
+      "task": "dashboard-implementation",
+      "status": "bailey-implementing",
+      "handoff_ready": "2024-01-16"
+    }
+  ],
+  "last_seen": "2024-01-15T11:30:00Z"
+}
+```
+
+## Implementation Architecture
+
+### Background Process Framework
+```typescript
+interface BackgroundProcess {
+  name: string;
+  interval: number; // milliseconds
+  async run(context: ProcessContext): Promise<void>;
+  async shouldRun(context: ProcessContext): Promise<boolean>;
+}
+
+class ProcessManager {
+  private processes: Map<string, BackgroundProcess>;
+  private memory: MemoryClient;
+  
+  async start() {
+    for (const [name, process] of this.processes) {
+      setInterval(async () => {
+        if (await process.shouldRun(this.context)) {
+          await process.run(this.context);
+        }
+      }, process.interval);
+    }
+  }
+}
+```
+
+### Memory Indexing Strategy
+1. **Write-through cache**: All observations written immediately to L2
+2. **Background indexing**: Processes run every 30s-5min depending on type
+3. **Smart batching**: Group related updates to minimize memory churn
+4. **Relevance scoring**: Continuously update scores based on access patterns
+5. **Compression**: Older entries compressed/summarized, recent kept detailed
+
+### Context Injection
+When building LLM prompts, the system will:
+1. Query active task state
+2. Include relevant code context (files, functions being worked on)
+3. Add recent research/documentation insights
+4. Include tool patterns for likely next actions
+5. Add cross-agent awareness if collaborating
+
+### Local Processing Power Usage
+- **Embedding generation**: Ollama with local models (no API calls)
+- **Pattern matching**: Rust-based processors for speed
+- **Index management**: LanceDB for vector operations
+- **File watching**: Native OS APIs for efficiency
+- **Git operations**: libgit2 bindings for speed
+
+## Benefits
+1. **Zero cognitive load**: Agents don't think about memory management
+2. **Rich context**: Every request includes highly relevant information
+3. **Learning system**: Gets better at predicting needed context over time
+4. **Collaborative**: Agents automatically aware of each other's work
+5. **Efficient**: Background processing keeps LLM calls focused
+
+## Next Steps
+1. Implement the background process framework in TypeScript
+2. Create the first processor: `code-indexer`
+3. Test with real agent workflows
+4. Add remaining processors incrementally
+5. Optimize based on dashboard metrics
+
+This system will make every Hivemind agent dramatically more capable without any changes to their prompts or behavior.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sesamespace/hivemind",
-  "version": "0.8.10",
+  "version": "0.8.11",
   "description": "Cognitive architecture for AI agents with multi-layered memory",
   "scripts": {
     "build": "tsup",
@@ -15,7 +15,9 @@
   },
   "dependencies": {
     "@sesamespace/sdk": "^0.1.6",
-    "smol-toml": "^1.6.0"
+    "smol-toml": "^1.6.0",
+    "axios": "^1.7.9",
+    "express": "^4.21.2"
   },
   "devDependencies": {
     "tsup": "^8.5.1"

package/src/memory/dashboard-integration.ts

@@ -0,0 +1,295 @@
+/**
+ * Dashboard Integration - Exposes memory system state to the Hivemind dashboard
+ */
+
+import { EventEmitter } from 'events';
+import express from 'express';
+import { MessageFlowIntegration } from './processors/message-flow-integration';
+import { ResearchDigester } from './processors/research-digester';
+import { CommandLearner } from './processors/command-learner';
+import { AgentSync } from './processors/agent-sync';
+
+export interface DashboardConfig {
+  port?: number;
+  messageFlow: MessageFlowIntegration;
+  researchDigester?: ResearchDigester;
+  commandLearner?: CommandLearner;
+  agentSync?: AgentSync;
+}
+
+export class DashboardIntegration extends EventEmitter {
+  private app: express.Application;
+  private messageFlow: MessageFlowIntegration;
+  private researchDigester?: ResearchDigester;
+  private commandLearner?: CommandLearner;
+  private agentSync?: AgentSync;
+
+  constructor(config: DashboardConfig) {
+    super();
+    
+    this.messageFlow = config.messageFlow;
+    this.researchDigester = config.researchDigester;
+    this.commandLearner = config.commandLearner;
+    this.agentSync = config.agentSync;
+
+    this.app = express();
+    this.setupRoutes();
+  }
+
+  private setupRoutes(): void {
+    this.app.use(express.json());
+
+    // Memory system overview
+    this.app.get('/api/memory/overview', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        const overview = {
+          ...state,
+          research: this.researchDigester ? {
+            entries: (await this.researchDigester['research']).size,
+            topics: Array.from(this.researchDigester['topicIndex'].keys())
+          } : null,
+          commands: this.commandLearner ? {
+            patterns: this.commandLearner['patterns'].size,
+            categories: Array.from(this.commandLearner['categoryIndex'].keys())
+          } : null,
+          agents: this.agentSync ? {
+            known: (await this.agentSync.getAgentKnowledge()).length,
+            sharedTasks: (await this.agentSync.getSharedTasks()).length
+          } : null
+        };
+        res.json(overview);
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Working set - currently active code files
+    this.app.get('/api/memory/working-set', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        res.json({
+          files: state.workingSet,
+          totalFiles: state.workingSet.length
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Active tasks
+    this.app.get('/api/memory/tasks', async (req, res) => {
+      try {
+        const state = await this.messageFlow.getState();
+        const sharedTasks = this.agentSync ? await this.agentSync.getSharedTasks() : [];
+        
+        res.json({
+          localTasks: state.tasks,
+          sharedTasks: sharedTasks,
+          total: state.tasks.length + sharedTasks.length
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Research entries
+    this.app.get('/api/memory/research', async (req, res) => {
+      if (!this.researchDigester) {
+        return res.json({ entries: [], topics: [] });
+      }
+
+      try {
+        const recent = await this.researchDigester.getRecent(20);
+        res.json({
+          entries: recent,
+          topics: Array.from(this.researchDigester['topicIndex'].keys())
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Search research
+    this.app.get('/api/memory/research/search', async (req, res) => {
+      if (!this.researchDigester) {
+        return res.json({ results: [] });
+      }
+
+      try {
+        const { q } = req.query;
+        const results = await this.researchDigester.search(q as string);
+        res.json({ results });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Command patterns
+    this.app.get('/api/memory/commands', async (req, res) => {
+      if (!this.commandLearner) {
+        return res.json({ patterns: [], categories: [] });
+      }
+
+      try {
+        const patterns = Array.from(this.commandLearner['patterns'].values());
+        const categories = Array.from(this.commandLearner['categoryIndex'].keys());
+        
+        res.json({
+          patterns: patterns.sort((a, b) => b.usageCount - a.usageCount),
+          categories
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Command suggestions
+    this.app.get('/api/memory/commands/suggest', async (req, res) => {
+      if (!this.commandLearner) {
+        return res.json({ suggestions: [] });
+      }
+
+      try {
+        const { context, category } = req.query;
+        const suggestions = await this.commandLearner.getSuggestions(
+          context as string || '',
+          category as string
+        );
+        res.json({ suggestions });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Agent knowledge
+    this.app.get('/api/memory/agents', async (req, res) => {
+      if (!this.agentSync) {
+        return res.json({ agents: [] });
+      }
+
+      try {
+        const agents = await this.agentSync.getAgentKnowledge();
+        res.json({ agents });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Memory timeline - recent activity
+    this.app.get('/api/memory/timeline', async (req, res) => {
+      try {
+        const timeline = [];
+        
+        // Add recent tasks
+        const state = await this.messageFlow.getState();
+        for (const task of state.tasks) {
+          timeline.push({
+            type: 'task',
+            timestamp: task.lastUpdate,
+            description: task.description,
+            metadata: { state: task.state }
+          });
+        }
+
+        // Add recent research
+        if (this.researchDigester) {
+          const research = await this.researchDigester.getRecent(5);
+          for (const entry of research) {
+            timeline.push({
+              type: 'research',
+              timestamp: entry.timestamp,
+              description: entry.title,
+              metadata: { url: entry.url, topics: entry.relatedTopics }
+            });
+          }
+        }
+
+        // Add recent commands
+        if (this.commandLearner) {
+          const patterns = Array.from(this.commandLearner['patterns'].values())
+            .sort((a, b) => b.lastUsed.getTime() - a.lastUsed.getTime())
+            .slice(0, 5);
+          
+          for (const pattern of patterns) {
+            timeline.push({
+              type: 'command',
+              timestamp: pattern.lastUsed,
+              description: pattern.description,
+              metadata: { pattern: pattern.pattern, category: pattern.category }
+            });
+          }
+        }
+
+        // Sort by timestamp
+        timeline.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+
+        res.json({ timeline: timeline.slice(0, 50) });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    // Context preview - what would be included in next request
+    this.app.post('/api/memory/context-preview', async (req, res) => {
+      try {
+        const { message } = req.body;
+        const context = await this.messageFlow.processMessage({
+          role: 'user',
+          content: message,
+          timestamp: new Date()
+        });
+
+        res.json({
+          context,
+          length: context.length,
+          sections: this.analyzeContextSections(context)
+        });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+  }
+
+  private analyzeContextSections(context: string): any[] {
+    const sections = [];
+    const lines = context.split('\n');
+    let currentSection = null;
+    let currentContent = [];
+
+    for (const line of lines) {
+      if (line.startsWith('## ')) {
+        if (currentSection) {
+          sections.push({
+            title: currentSection,
+            lines: currentContent.length,
+            characters: currentContent.join('\n').length
+          });
+        }
+        currentSection = line.substring(3);
+        currentContent = [];
+      } else {
+        currentContent.push(line);
+      }
+    }
+
+    if (currentSection) {
+      sections.push({
+        title: currentSection,
+        lines: currentContent.length,
+        characters: currentContent.join('\n').length
+      });
+    }
+
+    return sections;
+  }
+
+  async start(port: number = 9486): Promise<void> {
+    return new Promise((resolve) => {
+      this.app.listen(port, () => {
+        this.emit('started', { port });
+        resolve();
+      });
+    });
+  }
+}