@yamo/memory-mesh 2.1.3 → 2.3.0

package/README.md

@@ -12,6 +12,8 @@ Built on the [YAMO Protocol](https://github.com/yamo-protocol) for transparent a
 - **Portable CLI**: Simple JSON-based interface for any agent or language.
 - **YAMO Skills Integration**: Includes yamo-super workflow system with automatic memory learning.
 - **Pattern Recognition**: Workflows automatically store and retrieve execution patterns for optimization.
+- **LLM-Powered Reflections**: Generate insights from memories using configurable LLM providers.
+- **YAMO Audit Trail**: Automatic emission of structured blocks for all memory operations.
 
 ## Installation
 
@@ -29,9 +31,6 @@ memory-mesh store "My important memory" '{"tag":"test"}'
 
 # Search memories
 memory-mesh search "query" 5
-
-# Scrub content only
-scrubber scrub "Raw text content"
 ```
 
 ### Node.js API
@@ -44,6 +43,63 @@ await mesh.add('Content', { meta: 'data' });
 const results = await mesh.search('query');
 ```
 
+### Enhanced Reflections with LLM
+
+MemoryMesh supports LLM-powered reflection generation that synthesizes insights from stored memories:
+
+```javascript
+import { MemoryMesh } from '@yamo/memory-mesh';
+
+// Enable LLM integration (requires API key or local model)
+const mesh = new MemoryMesh({
+  enableLLM: true,
+  llmProvider: 'openai',  // or 'anthropic', 'ollama'
+  llmApiKey: process.env.OPENAI_API_KEY,
+  llmModel: 'gpt-4o-mini'
+});
+
+// Store some memories
+await mesh.add('Bug: type mismatch in keyword search', { type: 'bug' });
+await mesh.add('Bug: missing content field', { type: 'bug' });
+
+// Generate reflection (automatically stores result to memory)
+const reflection = await mesh.reflect({ topic: 'bugs', lookback: 10 });
+
+console.log(reflection.reflection);
+// Output: "Synthesized from 2 memories: Bug: type mismatch..., Bug: missing content..."
+
+console.log(reflection.confidence);  // 0.85
+console.log(reflection.yamoBlock);    // YAMO audit trail
+```
+
+**CLI Usage:**
+
+```bash
+# With LLM (default)
+memory-mesh reflect '{"topic": "bugs", "limit": 10}'
+
+# Without LLM (prompt-only for external LLM)
+memory-mesh reflect '{"topic": "bugs", "llm": false}'
+```
+
+### YAMO Audit Trail
+
+MemoryMesh automatically emits YAMO blocks for all operations when enabled:
+
+```javascript
+const mesh = new MemoryMesh({ enableYamo: true });
+
+// All operations now emit YAMO blocks
+await mesh.add('Memory content', { type: 'event' });      // emits 'retain' block
+await mesh.search('query');                                 // emits 'recall' block
+await mesh.reflect({ topic: 'test' });                      // emits 'reflect' block
+
+// Query YAMO log
+const yamoLog = await mesh.getYamoLog({ operationType: 'reflect', limit: 10 });
+console.log(yamoLog);
+// [{ id, agentId, operationType, yamoText, timestamp, ... }]
+```
+
 ## Using in a Project
 
 To use MemoryMesh with your Claude Code skills (like `yamo-super`) in a new project:
@@ -75,9 +131,6 @@ Your skills are now available in Claude Code with automatic memory integration:
 # Use yamo-super workflow system
 # Automatically retrieves similar past workflows and stores execution patterns
 claude /yamo-super
-
-# Use scrubber skill for content sanitization
-claude /scrubber content="raw text"
 ```
 
 **Memory Integration Features:**
@@ -87,7 +140,7 @@ claude /scrubber content="raw text"
 - **Review Phase**: Stores code review outcomes and quality metrics
 - **Complete Workflow**: Stores full execution pattern for future optimization
 
-YAMO agents will automatically find tools in `tools/memory_mesh.js` and `tools/scrubber.js`.
+YAMO agents will automatically find tools in `tools/memory_mesh.js`.
 
 ## Docker
 
@@ -127,3 +180,66 @@ Memory Mesh implements YAMO v2.1.0 compliance with:
 - **Development Guide**: [CLAUDE.md](CLAUDE.md) - Guide for Claude Code development
 - **Marketplace**: [.claude-plugin/marketplace.json](.claude-plugin/marketplace.json) - Plugin metadata
 
+## Configuration
+
+### LLM Provider Configuration
+
+```bash
+# Required for LLM-powered reflections
+LLM_PROVIDER=openai          # Provider: 'openai', 'anthropic', 'ollama'
+LLM_API_KEY=sk-...            # API key for OpenAI/Anthropic
+LLM_MODEL=gpt-4o-mini         # Model name
+LLM_BASE_URL=https://...      # Optional: Custom API base URL
+```
+
+**Supported Providers:**
+- **OpenAI**: GPT-4, GPT-4o-mini, etc.
+- **Anthropic**: Claude 3.5 Haiku, Sonnet, Opus
+- **Ollama**: Local models (llama3.2, mistral, etc.)
+
+### YAMO Configuration
+
+```bash
+# Optional YAMO settings
+ENABLE_YAMO=true              # Enable YAMO block emission (default: true)
+YAMO_DEBUG=true               # Enable verbose YAMO logging
+```
+
+### LanceDB Configuration
+
+```bash
+# Vector database settings
+LANCEDB_URI=./runtime/data/lancedb
+LANCEDB_MEMORY_TABLE=memory_entries
+```
+
+### Embedding Configuration
+
+```bash
+# Embedding model settings
+EMBEDDING_MODEL_TYPE=local    # 'local', 'openai', 'cohere', 'ollama'
+EMBEDDING_MODEL_NAME=Xenova/all-MiniLM-L6-v2
+EMBEDDING_DIMENSION=384
+```
+
+### Example .env File
+
+```bash
+# LLM for reflections
+LLM_PROVIDER=openai
+LLM_API_KEY=sk-your-key-here
+LLM_MODEL=gpt-4o-mini
+
+# YAMO audit
+ENABLE_YAMO=true
+YAMO_DEBUG=false
+
+# Vector DB
+LANCEDB_URI=./data/lancedb
+
+# Embeddings (local default)
+EMBEDDING_MODEL_TYPE=local
+EMBEDDING_MODEL_NAME=Xenova/all-MiniLM-L6-v2
+```
+
+

package/bin/setup.js

@@ -133,8 +133,7 @@ async function installTools() {
   }
 
   const toolFiles = [
-    { src: 'memory_mesh.js', name: 'Memory Mesh CLI' },
-    { src: 'scrubber.js', name: 'Scrubber CLI' }
+    { src: 'memory_mesh.js', name: 'Memory Mesh CLI' }
   ];
 
   let installed = 0;
@@ -169,7 +168,6 @@ function showUsage() {
 
   log('\n📚 Usage:', 'bright');
   log('  • Use /yamo-super in Claude or Gemini for workflow automation');
-  log('  • Use /scrubber skill for content sanitization');
   log('  • Call tools/memory_mesh.js for memory operations');
 
   log('\n🔗 Learn more:', 'bright');

package/lib/index.js

@@ -2,17 +2,5 @@
 export * from './lancedb/index.js';
 export * from './embeddings/index.js';
 export * from './search/index.js';
-export * from './privacy/index.js';
 export * from './memory/index.js';
 export * from './scrubber/index.js';
-export {
-  HandoffValidator,
-  Spinner,
-  ProgressBar,
-  MultiSpinner,
-  StreamingClient,
-  StreamingLLM,
-  sanitizeErrorForLogging,
-  withSanitizedErrors
-} from './utils/index.js';
-export * from './adapters/index.js';

package/lib/llm/client.js

@@ -0,0 +1,391 @@
+/**
+ * LLM Client - Multi-provider LLM API client for reflection generation
+ *
+ * Supports:
+ * - OpenAI (GPT-4, GPT-4o-mini, etc.)
+ * - Anthropic (Claude)
+ * - Ollama (local models)
+ * - Graceful fallback when LLM unavailable
+ */
+
+/**
+ * LLMClient provides unified interface for calling various LLM providers
+ * to generate reflections from memory contexts.
+ */
+export class LLMClient {
+  /**
+   * Create a new LLMClient instance
+   *
+   * @param {Object} [config={}] - Configuration options
+   * @param {string} [config.provider='openai'] - LLM provider ('openai', 'anthropic', 'ollama')
+   * @param {string} [config.apiKey] - API key (defaults to env var)
+   * @param {string} [config.model] - Model name
+   * @param {string} [config.baseUrl] - Base URL for API (optional)
+   * @param {number} [config.timeout=30000] - Request timeout in ms
+   * @param {number} [config.maxRetries=2] - Max retry attempts
+   */
+  constructor(config = {}) {
+    this.provider = config.provider || process.env.LLM_PROVIDER || 'openai';
+    this.apiKey = config.apiKey || process.env.LLM_API_KEY || '';
+    this.model = config.model || process.env.LLM_MODEL || this._getDefaultModel();
+    this.baseUrl = config.baseUrl || process.env.LLM_BASE_URL || this._getDefaultBaseUrl();
+    this.timeout = config.timeout || 30000;
+    this.maxRetries = config.maxRetries || 2;
+
+    // Statistics
+    this.stats = {
+      totalRequests: 0,
+      successfulRequests: 0,
+      failedRequests: 0,
+      fallbackCount: 0
+    };
+  }
+
+  /**
+   * Get default model for provider
+   * @private
+   * @returns {string} Default model name
+   */
+  _getDefaultModel() {
+    const defaults = {
+      openai: 'gpt-4o-mini',
+      anthropic: 'claude-3-5-haiku-20241022',
+      ollama: 'llama3.2'
+    };
+    return defaults[this.provider] || 'gpt-4o-mini';
+  }
+
+  /**
+   * Get default base URL for provider
+   * @private
+   * @returns {string} Default base URL
+   */
+  _getDefaultBaseUrl() {
+    const defaults = {
+      openai: 'https://api.openai.com/v1',
+      anthropic: 'https://api.anthropic.com/v1',
+      ollama: 'http://localhost:11434'
+    };
+    return defaults[this.provider] || 'https://api.openai.com/v1';
+  }
+
+  /**
+   * Generate reflection from memories
+   * Main entry point for reflection generation
+   *
+   * @param {string} prompt - The reflection prompt
+   * @param {Array} memories - Context memories
+   * @returns {Promise<Object>} { reflection, confidence }
+   */
+  async reflect(prompt, memories) {
+    this.stats.totalRequests++;
+
+    if (!memories || memories.length === 0) {
+      return this._fallback('No memories provided');
+    }
+
+    const systemPrompt = `You are a reflective AI agent. Review the provided memories and synthesize a high-level insight, belief, or observation.
+Respond ONLY in JSON format with exactly these keys:
+{
+  "reflection": "a concise insight or observation derived from the memories",
+  "confidence": 0.0 to 1.0
+}
+
+Keep the reflection brief (1-2 sentences) and actionable.`;
+
+    const userContent = this._formatMemoriesForLLM(prompt, memories);
+
+    try {
+      const response = await this._callWithRetry(systemPrompt, userContent);
+      const parsed = JSON.parse(response);
+
+      // Validate response structure
+      if (!parsed.reflection || typeof parsed.confidence !== 'number') {
+        throw new Error('Invalid LLM response format');
+      }
+
+      // Clamp confidence to valid range
+      parsed.confidence = Math.max(0, Math.min(1, parsed.confidence));
+
+      this.stats.successfulRequests++;
+      return parsed;
+
+    } catch (error) {
+      this.stats.failedRequests++;
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.warn(`[LLMClient] LLM call failed: ${errorMessage}`);
+      return this._fallback('LLM error', memories);
+    }
+  }
+
+  /**
+   * Format memories for LLM consumption
+   * @private
+   * @param {string} prompt - User prompt
+   * @param {Array} memories - Memory array
+   * @returns {string} Formatted content
+   */
+  _formatMemoriesForLLM(prompt, memories) {
+    const memoryList = memories
+      .map((m, i) => `${i + 1}. ${m.content}`)
+      .join('\n');
+
+    return `Prompt: ${prompt}\n\nMemories:\n${memoryList}\n\nBased on these memories, provide a brief reflective insight.`;
+  }
+
+  /**
+   * Call LLM with retry logic
+   * @private
+   * @param {string} systemPrompt - System prompt
+   * @param {string} userContent - User content
+   * @returns {Promise<string>} LLM response text
+   */
+  async _callWithRetry(systemPrompt, userContent) {
+    let lastError = null;
+
+    for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
+      try {
+        return await this._callLLM(systemPrompt, userContent);
+      } catch (error) {
+        lastError = error;
+        if (attempt < this.maxRetries) {
+          const delay = Math.pow(2, attempt) * 1000; // Exponential backoff
+          await this._sleep(delay);
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  /**
+   * Call LLM based on provider
+   * @private
+   * @param {string} systemPrompt - System prompt
+   * @param {string} userContent - User content
+   * @returns {Promise<string>} Response text
+   */
+  async _callLLM(systemPrompt, userContent) {
+    switch (this.provider) {
+      case 'openai':
+        return await this._callOpenAI(systemPrompt, userContent);
+      case 'anthropic':
+        return await this._callAnthropic(systemPrompt, userContent);
+      case 'ollama':
+        return await this._callOllama(systemPrompt, userContent);
+      default:
+        throw new Error(`Unsupported provider: ${this.provider}`);
+    }
+  }
+
+  /**
+   * Call OpenAI API
+   * @private
+   */
+  async _callOpenAI(systemPrompt, userContent) {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userContent }
+          ],
+          temperature: 0.7,
+          max_tokens: 500
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.choices[0].message.content;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Call Anthropic (Claude) API
+   * @private
+   */
+  async _callAnthropic(systemPrompt, userContent) {
+    if (!this.apiKey) {
+      throw new Error('Anthropic API key not configured');
+    }
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/messages`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.apiKey,
+          'anthropic-version': '2023-06-01'
+        },
+        body: JSON.stringify({
+          model: this.model,
+          max_tokens: 500,
+          system: systemPrompt,
+          messages: [
+            { role: 'user', content: userContent }
+          ]
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Anthropic API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.content[0].text;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Call Ollama (local) API
+   * @private
+   */
+  async _callOllama(systemPrompt, userContent) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/api/chat`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userContent }
+          ],
+          stream: false
+        }),
+        signal: controller.signal
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Ollama API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      return data.message.content;
+
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Request timeout');
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Fallback when LLM fails
+   * @private
+   * @param {string} reason - Fallback reason
+   * @param {Array} [memories=[]] - Memory array
+   * @returns {Object} Fallback result
+   */
+  _fallback(reason, memories = []) {
+    this.stats.fallbackCount++;
+
+    if (memories && memories.length > 0) {
+      // Simple aggregation fallback
+      const contents = memories.map(m => m.content);
+      const combined = contents.join('; ');
+      const preview = combined.length > 200
+        ? combined.substring(0, 200) + '...'
+        : combined;
+
+      return {
+        reflection: `Aggregated from ${memories.length} memories: ${preview}`,
+        confidence: 0.5
+      };
+    }
+
+    return {
+      reflection: `Reflection generation unavailable: ${reason}`,
+      confidence: 0.3
+    };
+  }
+
+  /**
+   * Sleep utility
+   * @private
+   * @param {number} ms - Milliseconds to sleep
+   * @returns {Promise<void>}
+   */
+  _sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Get client statistics
+   * @returns {Object} Statistics
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      successRate: this.stats.totalRequests > 0
+        ? (this.stats.successfulRequests / this.stats.totalRequests).toFixed(2)
+        : '0.00'
+    };
+  }
+
+  /**
+   * Reset statistics
+   */
+  resetStats() {
+    this.stats = {
+      totalRequests: 0,
+      successfulRequests: 0,
+      failedRequests: 0,
+      fallbackCount: 0
+    };
+  }
+}
+
+export default LLMClient;

package/lib/llm/index.js

@@ -0,0 +1,10 @@
+/**
+ * LLM Module - LLM client support for yamo-memory-mesh
+ * Exports multi-provider LLM client for reflection generation
+ */
+
+export { LLMClient } from './client.js';
+
+export default {
+  LLMClient: (await import('./client.js')).LLMClient
+};

package/lib/memory/index.js

@@ -1,3 +1 @@
-export { default as VectorMemory } from './vector-memory.js';
 export { default as MemoryMesh } from './memory-mesh.js';
-export { default as MigrateMemory } from './migrate-memory.js';