claude-flow 1.0.0

package/src/memory/backends/base.ts

@@ -0,0 +1,22 @@
+/**
+ * Base interface for memory backends
+ */
+
+import { MemoryEntry, MemoryQuery } from '../../utils/types.ts';
+
+export interface IMemoryBackend {
+  initialize(): Promise<void>;
+  shutdown(): Promise<void>;
+  store(entry: MemoryEntry): Promise<void>;
+  retrieve(id: string): Promise<MemoryEntry | undefined>;
+  update(id: string, entry: MemoryEntry): Promise<void>;
+  delete(id: string): Promise<void>;
+  query(query: MemoryQuery): Promise<MemoryEntry[]>;
+  getAllEntries(): Promise<MemoryEntry[]>;
+  getHealthStatus(): Promise<{ 
+    healthy: boolean; 
+    error?: string; 
+    metrics?: Record<string, number>;
+  }>;
+  performMaintenance?(): Promise<void>;
+}

package/src/memory/backends/markdown.ts

@@ -0,0 +1,283 @@
+/**
+ * Markdown backend implementation for human-readable memory storage
+ */
+
+import { IMemoryBackend } from './base.ts';
+import { MemoryEntry, MemoryQuery } from '../../utils/types.ts';
+import { ILogger } from '../../core/logger.ts';
+import { MemoryBackendError } from '../../utils/errors.ts';
+import { ensureDir } from 'https://deno.land/std@0.208.0/fs/mod.ts';
+
+/**
+ * Markdown-based memory backend
+ */
+export class MarkdownBackend implements IMemoryBackend {
+  private entries = new Map<string, MemoryEntry>();
+  private indexPath: string;
+
+  constructor(
+    private baseDir: string,
+    private logger: ILogger,
+  ) {
+    this.indexPath = `${this.baseDir}/index.json`;
+  }
+
+  async initialize(): Promise<void> {
+    this.logger.info('Initializing Markdown backend', { baseDir: this.baseDir });
+
+    try {
+      // Ensure directories exist
+      await ensureDir(this.baseDir);
+      await ensureDir(`${this.baseDir}/agents`);
+      await ensureDir(`${this.baseDir}/sessions`);
+
+      // Load index
+      await this.loadIndex();
+
+      this.logger.info('Markdown backend initialized');
+    } catch (error) {
+      throw new MemoryBackendError('Failed to initialize Markdown backend', { error });
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down Markdown backend');
+
+    // Save index before shutdown
+    await this.saveIndex();
+    this.entries.clear();
+  }
+
+  async store(entry: MemoryEntry): Promise<void> {
+    try {
+      // Store in memory
+      this.entries.set(entry.id, entry);
+
+      // Write to markdown file
+      await this.writeEntryToFile(entry);
+
+      // Update index
+      await this.saveIndex();
+    } catch (error) {
+      throw new MemoryBackendError('Failed to store entry', { error });
+    }
+  }
+
+  async retrieve(id: string): Promise<MemoryEntry | undefined> {
+    return this.entries.get(id);
+  }
+
+  async update(id: string, entry: MemoryEntry): Promise<void> {
+    if (!this.entries.has(id)) {
+      throw new MemoryBackendError(`Entry not found: ${id}`);
+    }
+
+    await this.store(entry);
+  }
+
+  async delete(id: string): Promise<void> {
+    const entry = this.entries.get(id);
+    if (!entry) {
+      return;
+    }
+
+    try {
+      // Delete from memory
+      this.entries.delete(id);
+
+      // Delete file
+      const filePath = this.getEntryFilePath(entry);
+      await Deno.remove(filePath);
+
+      // Update index
+      await this.saveIndex();
+    } catch (error) {
+      throw new MemoryBackendError('Failed to delete entry', { error });
+    }
+  }
+
+  async query(query: MemoryQuery): Promise<MemoryEntry[]> {
+    let results = Array.from(this.entries.values());
+
+    // Apply filters
+    if (query.agentId) {
+      results = results.filter(e => e.agentId === query.agentId);
+    }
+
+    if (query.sessionId) {
+      results = results.filter(e => e.sessionId === query.sessionId);
+    }
+
+    if (query.type) {
+      results = results.filter(e => e.type === query.type);
+    }
+
+    if (query.tags && query.tags.length > 0) {
+      results = results.filter(e => 
+        query.tags!.some(tag => e.tags.includes(tag)),
+      );
+    }
+
+    if (query.startTime) {
+      results = results.filter(e => 
+        e.timestamp.getTime() >= query.startTime!.getTime(),
+      );
+    }
+
+    if (query.endTime) {
+      results = results.filter(e => 
+        e.timestamp.getTime() <= query.endTime!.getTime(),
+      );
+    }
+
+    if (query.search) {
+      const searchLower = query.search.toLowerCase();
+      results = results.filter(e => 
+        e.content.toLowerCase().includes(searchLower) ||
+        e.tags.some(tag => tag.toLowerCase().includes(searchLower)),
+      );
+    }
+
+    // Sort by timestamp (newest first)
+    results.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+
+    // Apply pagination
+    const start = query.offset || 0;
+    const limit = query.limit || results.length;
+    results = results.slice(start, start + limit);
+
+    return results;
+  }
+
+  async getAllEntries(): Promise<MemoryEntry[]> {
+    return Array.from(this.entries.values());
+  }
+
+  async getHealthStatus(): Promise<{ 
+    healthy: boolean; 
+    error?: string; 
+    metrics?: Record<string, number>;
+  }> {
+    try {
+      // Check if directory is accessible
+      await Deno.stat(this.baseDir);
+
+      const entryCount = this.entries.size;
+      let totalSizeBytes = 0;
+
+      // Calculate total size
+      for (const entry of this.entries.values()) {
+        const filePath = this.getEntryFilePath(entry);
+        try {
+          const stat = await Deno.stat(filePath);
+          totalSizeBytes += stat.size;
+        } catch {
+          // File might not exist yet
+        }
+      }
+
+      return {
+        healthy: true,
+        metrics: {
+          entryCount,
+          totalSizeBytes,
+        },
+      };
+    } catch (error) {
+      return {
+        healthy: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      };
+    }
+  }
+
+  private async loadIndex(): Promise<void> {
+    try {
+      const content = await Deno.readTextFile(this.indexPath);
+      const index = JSON.parse(content) as Record<string, MemoryEntry>;
+
+      // Convert and validate entries
+      for (const [id, entry] of Object.entries(index)) {
+        // Reconstruct dates
+        entry.timestamp = new Date(entry.timestamp);
+        this.entries.set(id, entry);
+      }
+
+      this.logger.info('Loaded memory index', { entries: this.entries.size });
+    } catch (error) {
+      if (!(error instanceof Deno.errors.NotFound)) {
+        this.logger.warn('Failed to load index', { error });
+      }
+      // Start with empty index if file doesn't exist
+    }
+  }
+
+  private async saveIndex(): Promise<void> {
+    const index: Record<string, MemoryEntry> = {};
+    
+    for (const [id, entry] of this.entries) {
+      index[id] = entry;
+    }
+
+    const content = JSON.stringify(index, null, 2);
+    await Deno.writeTextFile(this.indexPath, content);
+  }
+
+  private async writeEntryToFile(entry: MemoryEntry): Promise<void> {
+    const filePath = this.getEntryFilePath(entry);
+    const dirPath = filePath.substring(0, filePath.lastIndexOf('/'));
+
+    // Ensure directory exists
+    await ensureDir(dirPath);
+
+    // Generate markdown content
+    const content = this.entryToMarkdown(entry);
+
+    // Write file
+    await Deno.writeTextFile(filePath, content);
+  }
+
+  private getEntryFilePath(entry: MemoryEntry): string {
+    const date = entry.timestamp.toISOString().split('T')[0];
+    const time = entry.timestamp.toISOString().split('T')[1].replace(/:/g, '-').split('.')[0];
+    
+    return `${this.baseDir}/agents/${entry.agentId}/${date}/${time}_${entry.id}.md`;
+  }
+
+  private entryToMarkdown(entry: MemoryEntry): string {
+    const lines: string[] = [
+      `# Memory Entry: ${entry.id}`,
+      '',
+      `**Agent**: ${entry.agentId}`,
+      `**Session**: ${entry.sessionId}`,
+      `**Type**: ${entry.type}`,
+      `**Timestamp**: ${entry.timestamp.toISOString()}`,
+      `**Version**: ${entry.version}`,
+      '',
+    ];
+
+    if (entry.parentId) {
+      lines.push(`**Parent**: ${entry.parentId}`, '');
+    }
+
+    if (entry.tags.length > 0) {
+      lines.push(`**Tags**: ${entry.tags.join(', ')}`, '');
+    }
+
+    lines.push('## Content', '', entry.content, '');
+
+    if (Object.keys(entry.context).length > 0) {
+      lines.push('## Context', '', '```json');
+      lines.push(JSON.stringify(entry.context, null, 2));
+      lines.push('```', '');
+    }
+
+    if (entry.metadata && Object.keys(entry.metadata).length > 0) {
+      lines.push('## Metadata', '', '```json');
+      lines.push(JSON.stringify(entry.metadata, null, 2));
+      lines.push('```', '');
+    }
+
+    return lines.join('\n');
+  }
+}

package/src/memory/backends/sqlite.ts

@@ -0,0 +1,329 @@
+/**
+ * SQLite backend implementation for memory storage
+ */
+
+import { IMemoryBackend } from './base.ts';
+import { MemoryEntry, MemoryQuery } from '../../utils/types.ts';
+import { ILogger } from '../../core/logger.ts';
+import { MemoryBackendError } from '../../utils/errors.ts';
+
+// SQLite bindings placeholder - in real implementation, use a proper SQLite library
+interface Database {
+  execute(sql: string, params?: unknown[]): Promise<unknown[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * SQLite-based memory backend
+ */
+export class SQLiteBackend implements IMemoryBackend {
+  private db?: Database;
+
+  constructor(
+    private dbPath: string,
+    private logger: ILogger,
+  ) {}
+
+  async initialize(): Promise<void> {
+    this.logger.info('Initializing SQLite backend', { dbPath: this.dbPath });
+
+    try {
+      // In real implementation, open SQLite connection
+      // this.db = await openDatabase(this.dbPath);
+
+      // Create tables
+      await this.createTables();
+
+      // Create indexes
+      await this.createIndexes();
+
+      this.logger.info('SQLite backend initialized');
+    } catch (error) {
+      throw new MemoryBackendError('Failed to initialize SQLite backend', { error });
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down SQLite backend');
+
+    if (this.db) {
+      await this.db.close();
+      delete this.db;
+    }
+  }
+
+  async store(entry: MemoryEntry): Promise<void> {
+    if (!this.db) {
+      throw new MemoryBackendError('Database not initialized');
+    }
+
+    const sql = `
+      INSERT OR REPLACE INTO memory_entries (
+        id, agent_id, session_id, type, content, 
+        context, timestamp, tags, version, parent_id, metadata
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `;
+
+    const params = [
+      entry.id,
+      entry.agentId,
+      entry.sessionId,
+      entry.type,
+      entry.content,
+      JSON.stringify(entry.context),
+      entry.timestamp.toISOString(),
+      JSON.stringify(entry.tags),
+      entry.version,
+      entry.parentId || null,
+      entry.metadata ? JSON.stringify(entry.metadata) : null,
+    ];
+
+    try {
+      await this.db.execute(sql, params);
+    } catch (error) {
+      throw new MemoryBackendError('Failed to store entry', { error });
+    }
+  }
+
+  async retrieve(id: string): Promise<MemoryEntry | undefined> {
+    if (!this.db) {
+      throw new MemoryBackendError('Database not initialized');
+    }
+
+    const sql = 'SELECT * FROM memory_entries WHERE id = ?';
+    
+    try {
+      const rows = await this.db.execute(sql, [id]);
+      
+      if (rows.length === 0) {
+        return undefined;
+      }
+
+      return this.rowToEntry(rows[0] as Record<string, unknown>);
+    } catch (error) {
+      throw new MemoryBackendError('Failed to retrieve entry', { error });
+    }
+  }
+
+  async update(id: string, entry: MemoryEntry): Promise<void> {
+    // SQLite INSERT OR REPLACE handles updates
+    await this.store(entry);
+  }
+
+  async delete(id: string): Promise<void> {
+    if (!this.db) {
+      throw new MemoryBackendError('Database not initialized');
+    }
+
+    const sql = 'DELETE FROM memory_entries WHERE id = ?';
+    
+    try {
+      await this.db.execute(sql, [id]);
+    } catch (error) {
+      throw new MemoryBackendError('Failed to delete entry', { error });
+    }
+  }
+
+  async query(query: MemoryQuery): Promise<MemoryEntry[]> {
+    if (!this.db) {
+      throw new MemoryBackendError('Database not initialized');
+    }
+
+    const conditions: string[] = [];
+    const params: unknown[] = [];
+
+    if (query.agentId) {
+      conditions.push('agent_id = ?');
+      params.push(query.agentId);
+    }
+
+    if (query.sessionId) {
+      conditions.push('session_id = ?');
+      params.push(query.sessionId);
+    }
+
+    if (query.type) {
+      conditions.push('type = ?');
+      params.push(query.type);
+    }
+
+    if (query.startTime) {
+      conditions.push('timestamp >= ?');
+      params.push(query.startTime.toISOString());
+    }
+
+    if (query.endTime) {
+      conditions.push('timestamp <= ?');
+      params.push(query.endTime.toISOString());
+    }
+
+    if (query.search) {
+      conditions.push('(content LIKE ? OR tags LIKE ?)');
+      params.push(`%${query.search}%`, `%${query.search}%`);
+    }
+
+    if (query.tags && query.tags.length > 0) {
+      const tagConditions = query.tags.map(() => 'tags LIKE ?');
+      conditions.push(`(${tagConditions.join(' OR ')})`);
+      query.tags.forEach(tag => params.push(`%"${tag}"%`));
+    }
+
+    let sql = 'SELECT * FROM memory_entries';
+    if (conditions.length > 0) {
+      sql += ' WHERE ' + conditions.join(' AND ');
+    }
+
+    sql += ' ORDER BY timestamp DESC';
+
+    if (query.limit) {
+      sql += ' LIMIT ?';
+      params.push(query.limit);
+    }
+
+    if (query.offset) {
+      sql += ' OFFSET ?';
+      params.push(query.offset);
+    }
+
+    try {
+      const rows = await this.db.execute(sql, params);
+      return rows.map(row => this.rowToEntry(row as Record<string, unknown>));
+    } catch (error) {
+      throw new MemoryBackendError('Failed to query entries', { error });
+    }
+  }
+
+  async getAllEntries(): Promise<MemoryEntry[]> {
+    if (!this.db) {
+      throw new MemoryBackendError('Database not initialized');
+    }
+
+    const sql = 'SELECT * FROM memory_entries ORDER BY timestamp DESC';
+    
+    try {
+      const rows = await this.db.execute(sql);
+      return rows.map(row => this.rowToEntry(row as Record<string, unknown>));
+    } catch (error) {
+      throw new MemoryBackendError('Failed to get all entries', { error });
+    }
+  }
+
+  async getHealthStatus(): Promise<{ 
+    healthy: boolean; 
+    error?: string; 
+    metrics?: Record<string, number>;
+  }> {
+    if (!this.db) {
+      return {
+        healthy: false,
+        error: 'Database not initialized',
+      };
+    }
+
+    try {
+      // Check database connectivity
+      await this.db.execute('SELECT 1');
+
+      // Get metrics
+      const [countResult] = await this.db.execute(
+        'SELECT COUNT(*) as count FROM memory_entries',
+      );
+      const entryCount = (countResult as any).count;
+
+      const [sizeResult] = await this.db.execute(
+        'SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()',
+      );
+      const dbSize = (sizeResult as any).size;
+
+      return {
+        healthy: true,
+        metrics: {
+          entryCount,
+          dbSizeBytes: dbSize,
+        },
+      };
+    } catch (error) {
+      return {
+        healthy: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      };
+    }
+  }
+
+  private async createTables(): Promise<void> {
+    const sql = `
+      CREATE TABLE IF NOT EXISTS memory_entries (
+        id TEXT PRIMARY KEY,
+        agent_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        type TEXT NOT NULL,
+        content TEXT NOT NULL,
+        context TEXT NOT NULL,
+        timestamp TEXT NOT NULL,
+        tags TEXT NOT NULL,
+        version INTEGER NOT NULL,
+        parent_id TEXT,
+        metadata TEXT,
+        created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+        updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+      )
+    `;
+
+    await this.db!.execute(sql);
+  }
+
+  private async createIndexes(): Promise<void> {
+    const indexes = [
+      'CREATE INDEX IF NOT EXISTS idx_agent_id ON memory_entries(agent_id)',
+      'CREATE INDEX IF NOT EXISTS idx_session_id ON memory_entries(session_id)',
+      'CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type)',
+      'CREATE INDEX IF NOT EXISTS idx_timestamp ON memory_entries(timestamp)',
+      'CREATE INDEX IF NOT EXISTS idx_parent_id ON memory_entries(parent_id)',
+    ];
+
+    for (const sql of indexes) {
+      await this.db!.execute(sql);
+    }
+  }
+
+  private rowToEntry(row: Record<string, unknown>): MemoryEntry {
+    const entry: MemoryEntry = {
+      id: row.id as string,
+      agentId: row.agent_id as string,
+      sessionId: row.session_id as string,
+      type: row.type as MemoryEntry['type'],
+      content: row.content as string,
+      context: JSON.parse(row.context as string),
+      timestamp: new Date(row.timestamp as string),
+      tags: JSON.parse(row.tags as string),
+      version: row.version as number,
+    };
+    
+    if (row.parent_id) {
+      entry.parentId = row.parent_id as string;
+    }
+    
+    if (row.metadata) {
+      entry.metadata = JSON.parse(row.metadata as string);
+    }
+    
+    return entry;
+  }
+}
+
+// Placeholder implementation - replace with actual SQLite library
+async function openDatabase(path: string): Promise<Database> {
+  // In real implementation, use a proper SQLite library like:
+  // - https://deno.land/x/sqlite
+  // - https://deno.land/x/sqlite3
+  
+  return {
+    async execute(sql: string, params?: unknown[]): Promise<unknown[]> {
+      // Placeholder
+      return [];
+    },
+    async close(): Promise<void> {
+      // Placeholder
+    },
+  };
+}