@yamo/memory-mesh 2.1.2 → 2.2.0

package/README.md

@@ -12,6 +12,8 @@ Built on the [YAMO Protocol](https://github.com/yamo-protocol) for transparent a
 - **Portable CLI**: Simple JSON-based interface for any agent or language.
 - **YAMO Skills Integration**: Includes yamo-super workflow system with automatic memory learning.
 - **Pattern Recognition**: Workflows automatically store and retrieve execution patterns for optimization.
+- **LLM-Powered Reflections**: Generate insights from memories using configurable LLM providers.
+- **YAMO Audit Trail**: Automatic emission of structured blocks for all memory operations.
 
 ## Installation
 
@@ -44,6 +46,63 @@ await mesh.add('Content', { meta: 'data' });
 const results = await mesh.search('query');
 ```
 
+### Enhanced Reflections with LLM
+
+MemoryMesh supports LLM-powered reflection generation that synthesizes insights from stored memories:
+
+```javascript
+import { MemoryMesh } from '@yamo/memory-mesh';
+
+// Enable LLM integration (requires API key or local model)
+const mesh = new MemoryMesh({
+  enableLLM: true,
+  llmProvider: 'openai',  // or 'anthropic', 'ollama'
+  llmApiKey: process.env.OPENAI_API_KEY,
+  llmModel: 'gpt-4o-mini'
+});
+
+// Store some memories
+await mesh.add('Bug: type mismatch in keyword search', { type: 'bug' });
+await mesh.add('Bug: missing content field', { type: 'bug' });
+
+// Generate reflection (automatically stores result to memory)
+const reflection = await mesh.reflect({ topic: 'bugs', lookback: 10 });
+
+console.log(reflection.reflection);
+// Output: "Synthesized from 2 memories: Bug: type mismatch..., Bug: missing content..."
+
+console.log(reflection.confidence);  // 0.85
+console.log(reflection.yamoBlock);    // YAMO audit trail
+```
+
+**CLI Usage:**
+
+```bash
+# With LLM (default)
+memory-mesh reflect '{"topic": "bugs", "limit": 10}'
+
+# Without LLM (prompt-only for external LLM)
+memory-mesh reflect '{"topic": "bugs", "llm": false}'
+```
+
+### YAMO Audit Trail
+
+MemoryMesh automatically emits YAMO blocks for all operations when enabled:
+
+```javascript
+const mesh = new MemoryMesh({ enableYamo: true });
+
+// All operations now emit YAMO blocks
+await mesh.add('Memory content', { type: 'event' });      // emits 'retain' block
+await mesh.search('query');                                 // emits 'recall' block
+await mesh.reflect({ topic: 'test' });                      // emits 'reflect' block
+
+// Query YAMO log
+const yamoLog = await mesh.getYamoLog({ operationType: 'reflect', limit: 10 });
+console.log(yamoLog);
+// [{ id, agentId, operationType, yamoText, timestamp, ... }]
+```
+
 ## Using in a Project
 
 To use MemoryMesh with your Claude Code skills (like `yamo-super`) in a new project:
@@ -127,3 +186,66 @@ Memory Mesh implements YAMO v2.1.0 compliance with:
 - **Development Guide**: [CLAUDE.md](CLAUDE.md) - Guide for Claude Code development
 - **Marketplace**: [.claude-plugin/marketplace.json](.claude-plugin/marketplace.json) - Plugin metadata
 
+## Configuration
+
+### LLM Provider Configuration
+
+```bash
+# Required for LLM-powered reflections
+LLM_PROVIDER=openai          # Provider: 'openai', 'anthropic', 'ollama'
+LLM_API_KEY=sk-...            # API key for OpenAI/Anthropic
+LLM_MODEL=gpt-4o-mini         # Model name
+LLM_BASE_URL=https://...      # Optional: Custom API base URL
+```
+
+**Supported Providers:**
+- **OpenAI**: GPT-4, GPT-4o-mini, etc.
+- **Anthropic**: Claude 3.5 Haiku, Sonnet, Opus
+- **Ollama**: Local models (llama3.2, mistral, etc.)
+
+### YAMO Configuration
+
+```bash
+# Optional YAMO settings
+ENABLE_YAMO=true              # Enable YAMO block emission (default: true)
+YAMO_DEBUG=true               # Enable verbose YAMO logging
+```
+
+### LanceDB Configuration
+
+```bash
+# Vector database settings
+LANCEDB_URI=./runtime/data/lancedb
+LANCEDB_MEMORY_TABLE=memory_entries
+```
+
+### Embedding Configuration
+
+```bash
+# Embedding model settings
+EMBEDDING_MODEL_TYPE=local    # 'local', 'openai', 'cohere', 'ollama'
+EMBEDDING_MODEL_NAME=Xenova/all-MiniLM-L6-v2
+EMBEDDING_DIMENSION=384
+```
+
+### Example .env File
+
+```bash
+# LLM for reflections
+LLM_PROVIDER=openai
+LLM_API_KEY=sk-your-key-here
+LLM_MODEL=gpt-4o-mini
+
+# YAMO audit
+ENABLE_YAMO=true
+YAMO_DEBUG=false
+
+# Vector DB
+LANCEDB_URI=./data/lancedb
+
+# Embeddings (local default)
+EMBEDDING_MODEL_TYPE=local
+EMBEDDING_MODEL_NAME=Xenova/all-MiniLM-L6-v2
+```
+
+

package/bin/memory_mesh.js

@@ -1,69 +1,14 @@
 #!/usr/bin/env node
 
 /**
- * MemoryMesh CLI Adapter
- * Provides a portable interface for skills to interact with the MemoryMesh system.
- * 
- * Usage:
- *   node tools/memory_mesh.js store <content_string_or_json> [metadata_json]
- *   node tools/memory_mesh.js search <query_string> [limit]
+ * MemoryMesh CLI Entry Point
+ * Delegates to the core CLI handler in lib/memory/memory-mesh.js
  */
 
-import { MemoryMesh } from '../lib/memory/memory-mesh.js';
-import path from 'path';
+import { run } from '../lib/memory/memory-mesh.js';
 
-// Parse arguments
-const args = process.argv.slice(2);
-const command = args[0];
-
-async function main() {
-    try {
-        const mesh = new MemoryMesh();
-        // Wait for initialization if necessary (MemoryMesh constructor usually doesn't await, 
-        // but operations might need internal init. We assume standard usage.)
-
-        if (command === 'store') {
-            const content = args[1];
-            let metadata = {};
-            if (args[2]) {
-                try {
-                    metadata = JSON.parse(args[2]);
-                } catch (e) {
-                    console.error(JSON.stringify({ error: "Invalid metadata JSON" }));
-                    process.exit(1);
-                }
-            }
-            
-            if (!content) {
-                console.error(JSON.stringify({ error: "Content required for store command" }));
-                process.exit(1);
-            }
-
-            const result = await mesh.add(content, metadata);
-            console.log(JSON.stringify({ success: true, id: result.id, message: "Memory stored successfully" }));
-            
-        } else if (command === 'search') {
-            const query = args[1];
-            const limit = parseInt(args[2]) || 5;
-            
-            if (!query) {
-                console.error(JSON.stringify({ error: "Query required for search command" }));
-                process.exit(1);
-            }
-
-            const results = await mesh.search(query, { limit });
-            console.log(JSON.stringify({ success: true, results: results }));
-            
-        } else {
-            console.error(JSON.stringify({ error: `Unknown command: ${command}` }));
-            console.error("Usage: node tools/memory_mesh.js [store|search] ...");
-            process.exit(1);
-        }
-
-    } catch (error) {
-        console.error(JSON.stringify({ error: error.message }));
-        process.exit(1);
-    }
-}
-
-main();
+// Execute the main CLI handler
+run().catch(err => {
+  console.error(`❌ Fatal Error: ${err.message}`);
+  process.exit(1);
+});

package/lib/embeddings/factory.js

@@ -7,10 +7,11 @@ import EmbeddingService from "./service.js";
 import { ConfigurationError, EmbeddingError } from "../lancedb/errors.js";
 
 class EmbeddingFactory {
-  constructor() {
+  constructor(ServiceClass = EmbeddingService) {
     this.primaryService = null;
     this.fallbackServices = [];
     this.configured = false;
+    this.ServiceClass = ServiceClass;
   }
 
   /**
@@ -22,10 +23,10 @@ class EmbeddingFactory {
     // Sort by priority (lower = higher priority)
     configs.sort((a, b) => a.priority - b.priority);
 
-    this.primaryService = new EmbeddingService(configs[0]);
+    this.primaryService = new this.ServiceClass(configs[0]);
 
     if (configs.length > 1) {
-      this.fallbackServices = configs.slice(1).map(c => new EmbeddingService(c));
+      this.fallbackServices = configs.slice(1).map(c => new this.ServiceClass(c));
     }
 
     this.configured = true;

package/lib/lancedb/client.js

@@ -28,6 +28,7 @@ class LanceDBClient {
    * @param {number} [config.maxRetries] - Maximum connection retries (default: 3)
    * @param {number} [config.retryDelay] - Delay between retries in ms (default: 1000)
    * @param {number} [config.vectorDimension] - Vector dimension for embeddings (default: 384)
+   * @param {Object} [config.driver] - LanceDB driver instance (for testing)
    */
   constructor(config = {}) {
     this.uri = (config && config.uri) || process.env.LANCEDB_URI || './data/lancedb';
@@ -35,6 +36,7 @@ class LanceDBClient {
     this.maxRetries = (config && config.maxRetries) || 3;
     this.retryDelay = (config && config.retryDelay) || 1000;
     this.vectorDimension = (config && config.vectorDimension) || DEFAULT_VECTOR_DIMENSION;
+    this.driver = (config && config.driver) || lancedb;
 
     // Connection state
     this.db = null;
@@ -66,7 +68,7 @@ class LanceDBClient {
         }
 
         // Connect to database
-        this.db = await lancedb.connect(this.uri);
+        this.db = await this.driver.connect(this.uri);
 
         // Initialize table with dynamic dimension (creates if doesn't exist, opens if it does)
         this.table = await createMemoryTableWithDimension(this.db, this.tableName, this.vectorDimension);

package/lib/llm/client.js

@@ -0,0 +1,391 @@
+/**
+ * LLM Client - Multi-provider LLM API client for reflection generation
+ *
+ * Supports:
+ * - OpenAI (GPT-4, GPT-4o-mini, etc.)
+ * - Anthropic (Claude)
+ * - Ollama (local models)
+ * - Graceful fallback when LLM unavailable
+ */
+
+/**
+ * LLMClient provides unified interface for calling various LLM providers
+ * to generate reflections from memory contexts.
+ */
+export class LLMClient {
+  /**
+   * Create a new LLMClient instance
+   *
+   * @param {Object} [config={}] - Configuration options
+   * @param {string} [config.provider='openai'] - LLM provider ('openai', 'anthropic', 'ollama')
+   * @param {string} [config.apiKey] - API key (defaults to env var)
+   * @param {string} [config.model] - Model name
+   * @param {string} [config.baseUrl] - Base URL for API (optional)
+   * @param {number} [config.timeout=30000] - Request timeout in ms
+   * @param {number} [config.maxRetries=2] - Max retry attempts
+   */
+  constructor(config = {}) {
+    this.provider = config.provider || process.env.LLM_PROVIDER || 'openai';
+    this.apiKey = config.apiKey || process.env.LLM_API_KEY || '';
+    this.model = config.model || process.env.LLM_MODEL || this._getDefaultModel();
+    this.baseUrl = config.baseUrl || process.env.LLM_BASE_URL || this._getDefaultBaseUrl();
+    this.timeout = config.timeout || 30000;
+    this.maxRetries = config.maxRetries || 2;
+
+    // Statistics
+    this.stats = {
+      totalRequests: 0,
+      successfulRequests: 0,
+      failedRequests: 0,
+      fallbackCount: 0
+    };
+  }
+
+  /**
+   * Get default model for provider
+   * @private
+   * @returns {string} Default model name
+   */
+  _getDefaultModel() {
+    const defaults = {
+      openai: 'gpt-4o-mini',
+      anthropic: 'claude-3-5-haiku-20241022',
+      ollama: 'llama3.2'
+    };
+    return defaults[this.provider] || 'gpt-4o-mini';
+  }
+
+  /**
+   * Get default base URL for provider
+   * @private
+   * @returns {string} Default base URL
+   */
+  _getDefaultBaseUrl() {
+    const defaults = {
+      openai: 'https://api.openai.com/v1',
+      anthropic: 'https://api.anthropic.com/v1',
+      ollama: 'http://localhost:11434'
+    };
+    return defaults[this.provider] || 'https://api.openai.com/v1';
+  }
+
+  /**
+   * Generate reflection from memories
+   * Main entry point for reflection generation
+   *
+   * @param {string} prompt - The reflection prompt
+   * @param {Array} memories - Context memories
+   * @returns {Promise<Object>} { reflection, confidence }
+   */
+  async reflect(prompt, memories) {
+    this.stats.totalRequests++;
+
+    if (!memories || memories.length === 0) {
+      return this._fallback('No memories provided');
+    }
+
+    const systemPrompt = `You are a reflective AI agent. Review the provided memories and synthesize a high-level insight, belief, or observation.
+Respond ONLY in JSON format with exactly these keys:
+{
+  "reflection": "a concise insight or observation derived from the memories",
+  "confidence": 0.0 to 1.0
+}
+
+Keep the reflection brief (1-2 sentences) and actionable.`;
+
+    const userContent = this._formatMemoriesForLLM(prompt, memories);
+
+    try {
+      const response = await this._callWithRetry(systemPrompt, userContent);
+      const parsed = JSON.parse(response);
+
+      // Validate response structure
+      if (!parsed.reflection || typeof parsed.confidence !== 'number') {
+        throw new Error('Invalid LLM response format');
+      }
+
+      // Clamp confidence to valid range
+      parsed.confidence = Math.max(0, Math.min(1, parsed.confidence));
+
+      this.stats.successfulRequests++;
+      return parsed;
+
+    } catch (error) {
+      this.stats.failedRequests++;
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.warn(`[LLMClient] LLM call failed: ${errorMessage}`);
+      return this._fallback('LLM error', memories);
+    }
+  }
+
+  /**
+   * Format memories for LLM consumption
+   * @private
+   * @param {string} prompt - User prompt
+   * @param {Array} memories - Memory array
+   * @returns {string} Formatted content
+   */
+  _formatMemoriesForLLM(prompt, memories) {
+    const memoryList = memories
+      .map((m, i) => `${i + 1}. ${m.content}`)
+      .join('\n');
+
+    return `Prompt: ${prompt}\n\nMemories:\n${memoryList}\n\nBased on these memories, provide a brief reflective insight.`;
+  }
+
+  /**
+   * Call LLM with retry logic
+   * @private
+   * @param {string} systemPrompt - System prompt
+   * @param {string} userContent - User content
+   * @returns {Promise<string>} LLM response text
+   */
+  async _callWithRetry(systemPrompt, userContent) {
+    let lastError = null;
+
+    for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
+      try {
+        return await this._callLLM(systemPrompt, userContent);
+      } catch (error) {
+        lastError = error;
+        if (attempt < this.maxRetries) {
+          const delay = Math.pow(2, attempt) * 1000; // Exponential backoff
+          await this._sleep(delay);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Call LLM based on provider
+   * @private
+   * @param {string} systemPrompt - System prompt
+   * @param {string} userContent - User content
+   * @returns {Promise<string>} Response text
+   */
+  async _callLLM(systemPrompt, userContent) {
+    switch (this.provider) {
+      case 'openai':
+        return await this._callOpenAI(systemPrompt, userContent);
+      case 'anthropic':
+        return await this._callAnthropic(systemPrompt, userContent);
+      case 'ollama':
+        return await this._callOllama(systemPrompt, userContent);
+      default:
+        throw new Error(`Unsupported provider: ${this.provider}`);
+    }
+  }
+
+  /**
+   * Call OpenAI API
+   * @private
+   */
+  async _callOpenAI(systemPrompt, userContent) {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userContent }
+          ],
+          temperature: 0.7,
+          max_tokens: 500
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.choices[0].message.content;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Call Anthropic (Claude) API
+   * @private
+   */
+  async _callAnthropic(systemPrompt, userContent) {
+    if (!this.apiKey) {
+      throw new Error('Anthropic API key not configured');
+    }
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/messages`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.apiKey,
+          'anthropic-version': '2023-06-01'
+        },
+        body: JSON.stringify({
+          model: this.model,
+          max_tokens: 500,
+          system: systemPrompt,
+          messages: [
+            { role: 'user', content: userContent }
+          ]
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Anthropic API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.content[0].text;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Call Ollama (local) API
+   * @private
+   */
+  async _callOllama(systemPrompt, userContent) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/api/chat`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userContent }
+          ],
+          stream: false
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Ollama API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.message.content;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Fallback when LLM fails
+   * @private
+   * @param {string} reason - Fallback reason
+   * @param {Array} [memories=[]] - Memory array
+   * @returns {Object} Fallback result
+   */
+  _fallback(reason, memories = []) {
+    this.stats.fallbackCount++;
+
+    if (memories && memories.length > 0) {
+      // Simple aggregation fallback
+      const contents = memories.map(m => m.content);
+      const combined = contents.join('; ');
+      const preview = combined.length > 200
+        ? combined.substring(0, 200) + '...'
+        : combined;
+
+      return {
+        reflection: `Aggregated from ${memories.length} memories: ${preview}`,
+        confidence: 0.5
+      };
+    }
+
+    return {
+      reflection: `Reflection generation unavailable: ${reason}`,
+      confidence: 0.3
+    };
+  }
+
+  /**
+   * Sleep utility
+   * @private
+   * @param {number} ms - Milliseconds to sleep
+   * @returns {Promise<void>}
+   */
+  _sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Get client statistics
+   * @returns {Object} Statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      successRate: this.stats.totalRequests > 0
+        ? (this.stats.successfulRequests / this.stats.totalRequests).toFixed(2)
+        : '0.00'
+    };
+  }
+
+  /**
+   * Reset statistics
+   */
+  resetStats() {
+    this.stats = {
+      totalRequests: 0,
+      successfulRequests: 0,
+      failedRequests: 0,
+      fallbackCount: 0
+    };
+  }
+}
+
+export default LLMClient;

package/lib/llm/index.js

@@ -0,0 +1,10 @@
+/**
+ * LLM Module - LLM client support for yamo-memory-mesh
+ * Exports multi-provider LLM client for reflection generation
+ */
+
+export { LLMClient } from './client.js';
+
+export default {
+  LLMClient: (await import('./client.js')).LLMClient
+};