@claude-flow/memory 3.0.0-alpha.1

package/src/migration.ts

@@ -0,0 +1,669 @@
+/**
+ * V3 Memory Migration Utility
+ *
+ * Migrates data from legacy memory systems (SQLite, Markdown, JSON, etc.)
+ * to the unified AgentDB-backed memory system with HNSW indexing.
+ *
+ * @module v3/memory/migration
+ */
+
+import { EventEmitter } from 'node:events';
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import {
+  MigrationConfig,
+  MigrationProgress,
+  MigrationResult,
+  MigrationError,
+  MigrationSource,
+  MemoryEntry,
+  MemoryType,
+  MemoryEntryInput,
+  EmbeddingGenerator,
+  createDefaultEntry,
+} from './types.js';
+import { AgentDBAdapter } from './agentdb-adapter.js';
+
+/**
+ * Default migration configuration
+ */
+const DEFAULT_MIGRATION_CONFIG: Partial<MigrationConfig> = {
+  batchSize: 100,
+  generateEmbeddings: true,
+  validateData: true,
+  continueOnError: true,
+};
+
+/**
+ * Legacy entry format (common structure)
+ */
+interface LegacyEntry {
+  id?: string;
+  key: string;
+  value: unknown;
+  namespace?: string;
+  tags?: string[];
+  metadata?: Record<string, unknown>;
+  timestamp?: number;
+  createdAt?: string | number;
+  updatedAt?: string | number;
+  created_at?: string | number;
+  updated_at?: string | number;
+}
+
+/**
+ * Memory Migration Manager
+ *
+ * Handles migration from:
+ * - SQLite backends (.db files)
+ * - Markdown backends (.md files)
+ * - JSON memory stores (.json files)
+ * - MemoryManager instances
+ * - SwarmMemory instances
+ * - DistributedMemory instances
+ */
+export class MemoryMigrator extends EventEmitter {
+  private config: MigrationConfig;
+  private target: AgentDBAdapter;
+  private embeddingGenerator?: EmbeddingGenerator;
+  private progress: MigrationProgress;
+
+  constructor(
+    target: AgentDBAdapter,
+    config: Partial<MigrationConfig>,
+    embeddingGenerator?: EmbeddingGenerator
+  ) {
+    super();
+    this.target = target;
+    this.config = { ...DEFAULT_MIGRATION_CONFIG, ...config } as MigrationConfig;
+    this.embeddingGenerator = embeddingGenerator;
+    this.progress = this.initializeProgress();
+  }
+
+  /**
+   * Run the migration
+   */
+  async migrate(): Promise<MigrationResult> {
+    const startTime = Date.now();
+    this.progress = this.initializeProgress();
+
+    this.emit('migration:started', { source: this.config.source });
+
+    try {
+      // Load entries from source
+      const entries = await this.loadFromSource();
+      this.progress.total = entries.length;
+      this.progress.totalBatches = Math.ceil(entries.length / this.config.batchSize);
+
+      this.emit('migration:progress', { ...this.progress });
+
+      // Process in batches
+      for (let i = 0; i < entries.length; i += this.config.batchSize) {
+        const batch = entries.slice(i, i + this.config.batchSize);
+        this.progress.currentBatch = Math.floor(i / this.config.batchSize) + 1;
+
+        await this.processBatch(batch);
+
+        this.progress.percentage = Math.round(
+          (this.progress.migrated / this.progress.total) * 100
+        );
+        this.progress.estimatedTimeRemaining = this.estimateTimeRemaining(
+          startTime,
+          this.progress.migrated,
+          this.progress.total
+        );
+
+        this.emit('migration:progress', { ...this.progress });
+      }
+
+      const duration = Date.now() - startTime;
+
+      const result: MigrationResult = {
+        success: this.progress.failed === 0 || this.config.continueOnError,
+        progress: { ...this.progress },
+        duration,
+        summary: this.generateSummary(),
+      };
+
+      this.emit('migration:completed', result);
+      return result;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+
+      const result: MigrationResult = {
+        success: false,
+        progress: { ...this.progress },
+        duration,
+        summary: `Migration failed: ${(error as Error).message}`,
+      };
+
+      this.emit('migration:failed', { error, result });
+      return result;
+    }
+  }
+
+  /**
+   * Get current migration progress
+   */
+  getProgress(): MigrationProgress {
+    return { ...this.progress };
+  }
+
+  // ===== Source Loaders =====
+
+  private async loadFromSource(): Promise<LegacyEntry[]> {
+    switch (this.config.source) {
+      case 'sqlite':
+        return this.loadFromSQLite();
+      case 'markdown':
+        return this.loadFromMarkdown();
+      case 'json':
+        return this.loadFromJSON();
+      case 'memory-manager':
+        return this.loadFromMemoryManager();
+      case 'swarm-memory':
+        return this.loadFromSwarmMemory();
+      case 'distributed-memory':
+        return this.loadFromDistributedMemory();
+      default:
+        throw new Error(`Unknown migration source: ${this.config.source}`);
+    }
+  }
+
+  private async loadFromSQLite(): Promise<LegacyEntry[]> {
+    const entries: LegacyEntry[] = [];
+    const dbPath = this.config.sourcePath;
+
+    try {
+      // Dynamic import for better-sqlite3 or similar
+      // In production, would use actual SQLite library
+      const fileContent = await fs.readFile(dbPath);
+
+      // Parse SQLite format (simplified - actual implementation would use SQLite library)
+      // For now, we'll try to read it as a JSON export format
+      if (dbPath.endsWith('.json')) {
+        const data = JSON.parse(fileContent.toString());
+        if (Array.isArray(data)) {
+          return data;
+        } else if (data.entries) {
+          return data.entries;
+        }
+      }
+
+      // SQLite parsing would go here using better-sqlite3 or sql.js
+      this.emit('migration:warning', {
+        message: 'Direct SQLite parsing requires additional setup. Using export format.',
+      });
+
+      return entries;
+    } catch (error) {
+      throw new Error(`Failed to load SQLite: ${(error as Error).message}`);
+    }
+  }
+
+  private async loadFromMarkdown(): Promise<LegacyEntry[]> {
+    const entries: LegacyEntry[] = [];
+    const basePath = this.config.sourcePath;
+
+    try {
+      const files = await this.walkDirectory(basePath, '.md');
+
+      for (const filePath of files) {
+        try {
+          const content = await fs.readFile(filePath, 'utf-8');
+          const entry = this.parseMarkdownEntry(filePath, content, basePath);
+          if (entry) {
+            entries.push(entry);
+          }
+        } catch (error) {
+          this.addError(filePath, (error as Error).message, 'PARSE_ERROR', true);
+        }
+      }
+
+      return entries;
+    } catch (error) {
+      throw new Error(`Failed to load Markdown: ${(error as Error).message}`);
+    }
+  }
+
+  private async loadFromJSON(): Promise<LegacyEntry[]> {
+    const filePath = this.config.sourcePath;
+
+    try {
+      const content = await fs.readFile(filePath, 'utf-8');
+      const data = JSON.parse(content);
+
+      // Handle different JSON formats
+      if (Array.isArray(data)) {
+        return data;
+      } else if (data.entries) {
+        return data.entries;
+      } else if (typeof data === 'object') {
+        // Assume it's a namespace -> entries map
+        const entries: LegacyEntry[] = [];
+        for (const [namespace, namespaceEntries] of Object.entries(data)) {
+          if (Array.isArray(namespaceEntries)) {
+            for (const entry of namespaceEntries) {
+              entries.push({ ...entry, namespace });
+            }
+          }
+        }
+        return entries;
+      }
+
+      return [];
+    } catch (error) {
+      throw new Error(`Failed to load JSON: ${(error as Error).message}`);
+    }
+  }
+
+  private async loadFromMemoryManager(): Promise<LegacyEntry[]> {
+    // Would integrate with existing MemoryManager instance
+    // For now, try to load from common paths
+    const possiblePaths = [
+      './memory/memory-store.json',
+      './.swarm/memory.db',
+      './memory.json',
+    ];
+
+    for (const p of possiblePaths) {
+      try {
+        const fullPath = path.resolve(this.config.sourcePath, p);
+        await fs.access(fullPath);
+        return this.loadFromJSON();
+      } catch {
+        continue;
+      }
+    }
+
+    return [];
+  }
+
+  private async loadFromSwarmMemory(): Promise<LegacyEntry[]> {
+    // Would integrate with SwarmMemory partitions
+    const entries: LegacyEntry[] = [];
+    const basePath = this.config.sourcePath;
+
+    try {
+      // Check for swarm memory directory structure
+      const partitionsPath = path.join(basePath, '.swarm', 'memory');
+      const files = await this.walkDirectory(partitionsPath, '.json');
+
+      for (const filePath of files) {
+        try {
+          const content = await fs.readFile(filePath, 'utf-8');
+          const data = JSON.parse(content);
+
+          // Extract namespace from file path
+          const relativePath = path.relative(partitionsPath, filePath);
+          const namespace = path.dirname(relativePath).replace(/\\/g, '/');
+
+          if (Array.isArray(data)) {
+            entries.push(...data.map((e: LegacyEntry) => ({ ...e, namespace })));
+          } else if (data.entries) {
+            entries.push(...data.entries.map((e: LegacyEntry) => ({ ...e, namespace })));
+          }
+        } catch (error) {
+          this.addError(filePath, (error as Error).message, 'PARSE_ERROR', true);
+        }
+      }
+
+      return entries;
+    } catch (error) {
+      return [];
+    }
+  }
+
+  private async loadFromDistributedMemory(): Promise<LegacyEntry[]> {
+    // Would integrate with DistributedMemorySystem nodes
+    return this.loadFromSwarmMemory(); // Similar structure
+  }
+
+  // ===== Batch Processing =====
+
+  private async processBatch(batch: LegacyEntry[]): Promise<void> {
+    for (const legacyEntry of batch) {
+      try {
+        // Validate if enabled
+        if (this.config.validateData) {
+          const validation = this.validateEntry(legacyEntry);
+          if (!validation.valid) {
+            if (this.config.continueOnError) {
+              this.addError(
+                legacyEntry.key || 'unknown',
+                validation.reason || 'Validation failed',
+                'VALIDATION_ERROR',
+                false
+              );
+              this.progress.skipped++;
+              continue;
+            } else {
+              throw new Error(validation.reason);
+            }
+          }
+        }
+
+        // Transform to new format
+        const newEntry = await this.transformEntry(legacyEntry);
+
+        // Store in target
+        await this.target.store(newEntry);
+        this.progress.migrated++;
+      } catch (error) {
+        if (this.config.continueOnError) {
+          this.addError(
+            legacyEntry.key || 'unknown',
+            (error as Error).message,
+            'STORE_ERROR',
+            true
+          );
+          this.progress.failed++;
+        } else {
+          throw error;
+        }
+      }
+    }
+  }
+
+  private async transformEntry(legacy: LegacyEntry): Promise<MemoryEntry> {
+    // Map namespace if configured
+    let namespace = legacy.namespace || 'default';
+    if (this.config.namespaceMapping && this.config.namespaceMapping[namespace]) {
+      namespace = this.config.namespaceMapping[namespace];
+    }
+
+    // Determine content
+    const content =
+      typeof legacy.value === 'string'
+        ? legacy.value
+        : JSON.stringify(legacy.value);
+
+    // Map type if configured
+    let type: MemoryType = 'semantic';
+    if (legacy.metadata?.type && typeof legacy.metadata.type === 'string') {
+      if (this.config.typeMapping && this.config.typeMapping[legacy.metadata.type]) {
+        type = this.config.typeMapping[legacy.metadata.type];
+      } else if (this.isValidMemoryType(legacy.metadata.type)) {
+        type = legacy.metadata.type as MemoryType;
+      }
+    }
+
+    // Parse timestamps
+    const createdAt = this.parseTimestamp(
+      legacy.createdAt || legacy.created_at || legacy.timestamp
+    );
+    const updatedAt = this.parseTimestamp(
+      legacy.updatedAt || legacy.updated_at || legacy.timestamp
+    );
+
+    const input: MemoryEntryInput = {
+      key: legacy.key,
+      content,
+      type,
+      namespace,
+      tags: legacy.tags || [],
+      metadata: {
+        ...legacy.metadata,
+        migrated: true,
+        migrationSource: this.config.source,
+        migrationTimestamp: Date.now(),
+        originalValue: legacy.value,
+      },
+    };
+
+    const entry = createDefaultEntry(input);
+    entry.createdAt = createdAt;
+    entry.updatedAt = updatedAt;
+
+    // Generate embedding if configured
+    if (this.config.generateEmbeddings && this.embeddingGenerator) {
+      try {
+        entry.embedding = await this.embeddingGenerator(content);
+      } catch (error) {
+        // Log but don't fail
+        this.emit('migration:warning', {
+          message: `Failed to generate embedding for ${legacy.key}: ${(error as Error).message}`,
+        });
+      }
+    }
+
+    return entry;
+  }
+
+  // ===== Helper Methods =====
+
+  private initializeProgress(): MigrationProgress {
+    return {
+      total: 0,
+      migrated: 0,
+      failed: 0,
+      skipped: 0,
+      currentBatch: 0,
+      totalBatches: 0,
+      percentage: 0,
+      estimatedTimeRemaining: 0,
+      errors: [],
+    };
+  }
+
+  private validateEntry(entry: LegacyEntry): { valid: boolean; reason?: string } {
+    if (!entry.key || typeof entry.key !== 'string') {
+      return { valid: false, reason: 'Missing or invalid key' };
+    }
+
+    if (entry.value === undefined) {
+      return { valid: false, reason: 'Missing value' };
+    }
+
+    if (entry.key.length > 500) {
+      return { valid: false, reason: 'Key too long (max 500 chars)' };
+    }
+
+    return { valid: true };
+  }
+
+  private addError(
+    entryId: string,
+    message: string,
+    code: string,
+    recoverable: boolean
+  ): void {
+    const error: MigrationError = {
+      entryId,
+      message,
+      code,
+      recoverable,
+    };
+    this.progress.errors.push(error);
+    this.emit('migration:error', error);
+  }
+
+  private parseTimestamp(value: string | number | undefined): number {
+    if (!value) return Date.now();
+
+    if (typeof value === 'number') {
+      // Handle both milliseconds and seconds
+      return value > 1e12 ? value : value * 1000;
+    }
+
+    const parsed = Date.parse(value);
+    return isNaN(parsed) ? Date.now() : parsed;
+  }
+
+  private isValidMemoryType(type: string): boolean {
+    return ['episodic', 'semantic', 'procedural', 'working', 'cache'].includes(type);
+  }
+
+  private estimateTimeRemaining(
+    startTime: number,
+    completed: number,
+    total: number
+  ): number {
+    if (completed === 0) return 0;
+
+    const elapsed = Date.now() - startTime;
+    const rate = completed / elapsed;
+    const remaining = total - completed;
+
+    return Math.round(remaining / rate);
+  }
+
+  private generateSummary(): string {
+    const { migrated, failed, skipped, total, errors } = this.progress;
+
+    let summary = `Migrated ${migrated}/${total} entries`;
+
+    if (failed > 0) {
+      summary += `, ${failed} failed`;
+    }
+
+    if (skipped > 0) {
+      summary += `, ${skipped} skipped`;
+    }
+
+    if (errors.length > 0) {
+      const errorTypes = new Map<string, number>();
+      for (const error of errors) {
+        errorTypes.set(error.code, (errorTypes.get(error.code) || 0) + 1);
+      }
+
+      const errorSummary = Array.from(errorTypes.entries())
+        .map(([code, count]) => `${code}: ${count}`)
+        .join(', ');
+
+      summary += `. Errors: ${errorSummary}`;
+    }
+
+    return summary;
+  }
+
+  private async walkDirectory(dir: string, extension: string): Promise<string[]> {
+    const files: string[] = [];
+
+    try {
+      const entries = await fs.readdir(dir, { withFileTypes: true });
+
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+
+        if (entry.isDirectory()) {
+          const subFiles = await this.walkDirectory(fullPath, extension);
+          files.push(...subFiles);
+        } else if (entry.isFile() && entry.name.endsWith(extension)) {
+          files.push(fullPath);
+        }
+      }
+    } catch (error) {
+      // Directory doesn't exist or isn't readable
+    }
+
+    return files;
+  }
+
+  private parseMarkdownEntry(
+    filePath: string,
+    content: string,
+    basePath: string
+  ): LegacyEntry | null {
+    // Extract frontmatter if present
+    const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+
+    let metadata: Record<string, unknown> = {};
+    let body = content;
+
+    if (frontmatterMatch) {
+      try {
+        // Simple YAML-like parsing
+        const frontmatter = frontmatterMatch[1];
+        for (const line of frontmatter.split('\n')) {
+          const colonIndex = line.indexOf(':');
+          if (colonIndex > 0) {
+            const key = line.substring(0, colonIndex).trim();
+            let value: unknown = line.substring(colonIndex + 1).trim();
+
+            // Parse common types
+            if (value === 'true') value = true;
+            else if (value === 'false') value = false;
+            else if (typeof value === 'string' && /^\d+$/.test(value)) value = parseInt(value, 10);
+            else if (typeof value === 'string' && value.startsWith('[') && value.endsWith(']')) {
+              try {
+                value = JSON.parse(value.replace(/'/g, '"'));
+              } catch {
+                // Keep as string
+              }
+            }
+
+            metadata[key] = value;
+          }
+        }
+        body = frontmatterMatch[2];
+      } catch {
+        // Failed to parse frontmatter, use whole content
+      }
+    }
+
+    // Derive key from file path
+    const relativePath = path.relative(basePath, filePath);
+    const key = relativePath
+      .replace(/\\/g, '/')
+      .replace(/\.md$/, '')
+      .replace(/\//g, ':');
+
+    // Derive namespace from directory structure
+    const namespace = path.dirname(relativePath).replace(/\\/g, '/') || 'default';
+
+    return {
+      key,
+      value: body.trim(),
+      namespace,
+      tags: Array.isArray(metadata.tags) ? metadata.tags : [],
+      metadata,
+      timestamp: Date.now(),
+    };
+  }
+}
+
+/**
+ * Convenience function to create a migrator
+ */
+export function createMigrator(
+  target: AgentDBAdapter,
+  source: MigrationSource,
+  sourcePath: string,
+  options: Partial<MigrationConfig> = {},
+  embeddingGenerator?: EmbeddingGenerator
+): MemoryMigrator {
+  return new MemoryMigrator(
+    target,
+    { source, sourcePath, ...options },
+    embeddingGenerator
+  );
+}
+
+/**
+ * Migrate from multiple sources
+ */
+export async function migrateMultipleSources(
+  target: AgentDBAdapter,
+  sources: Array<{ source: MigrationSource; path: string }>,
+  options: Partial<MigrationConfig> = {},
+  embeddingGenerator?: EmbeddingGenerator
+): Promise<MigrationResult[]> {
+  const results: MigrationResult[] = [];
+
+  for (const { source, path: sourcePath } of sources) {
+    const migrator = createMigrator(
+      target,
+      source,
+      sourcePath,
+      options,
+      embeddingGenerator
+    );
+    const result = await migrator.migrate();
+    results.push(result);
+  }
+
+  return results;
+}
+
+export default MemoryMigrator;