moflo 4.8.16 → 4.8.17

package/src/@claude-flow/memory/dist/migration.js

@@ -1,513 +0,0 @@
-/**
- * V3 Memory Migration Utility
- *
- * Migrates data from legacy memory systems (SQLite, Markdown, JSON, etc.)
- * to the unified AgentDB-backed memory system with HNSW indexing.
- *
- * @module v3/memory/migration
- */
-import { EventEmitter } from 'node:events';
-import { promises as fs } from 'node:fs';
-import * as path from 'node:path';
-import { createDefaultEntry, } from './types.js';
-/**
- * Default migration configuration
- */
-const DEFAULT_MIGRATION_CONFIG = {
-    batchSize: 100,
-    generateEmbeddings: true,
-    validateData: true,
-    continueOnError: true,
-};
-/**
- * Memory Migration Manager
- *
- * Handles migration from:
- * - SQLite backends (.db files)
- * - Markdown backends (.md files)
- * - JSON memory stores (.json files)
- * - MemoryManager instances
- * - SwarmMemory instances
- * - DistributedMemory instances
- */
-export class MemoryMigrator extends EventEmitter {
-    config;
-    target;
-    embeddingGenerator;
-    progress;
-    constructor(target, config, embeddingGenerator) {
-        super();
-        this.target = target;
-        this.config = { ...DEFAULT_MIGRATION_CONFIG, ...config };
-        this.embeddingGenerator = embeddingGenerator;
-        this.progress = this.initializeProgress();
-    }
-    /**
-     * Run the migration
-     */
-    async migrate() {
-        const startTime = Date.now();
-        this.progress = this.initializeProgress();
-        this.emit('migration:started', { source: this.config.source });
-        try {
-            // Load entries from source
-            const entries = await this.loadFromSource();
-            this.progress.total = entries.length;
-            this.progress.totalBatches = Math.ceil(entries.length / this.config.batchSize);
-            this.emit('migration:progress', { ...this.progress });
-            // Process in batches
-            for (let i = 0; i < entries.length; i += this.config.batchSize) {
-                const batch = entries.slice(i, i + this.config.batchSize);
-                this.progress.currentBatch = Math.floor(i / this.config.batchSize) + 1;
-                await this.processBatch(batch);
-                this.progress.percentage = Math.round((this.progress.migrated / this.progress.total) * 100);
-                this.progress.estimatedTimeRemaining = this.estimateTimeRemaining(startTime, this.progress.migrated, this.progress.total);
-                this.emit('migration:progress', { ...this.progress });
-            }
-            const duration = Date.now() - startTime;
-            const result = {
-                success: this.progress.failed === 0 || this.config.continueOnError,
-                progress: { ...this.progress },
-                duration,
-                summary: this.generateSummary(),
-            };
-            this.emit('migration:completed', result);
-            return result;
-        }
-        catch (error) {
-            const duration = Date.now() - startTime;
-            const result = {
-                success: false,
-                progress: { ...this.progress },
-                duration,
-                summary: `Migration failed: ${error.message}`,
-            };
-            this.emit('migration:failed', { error, result });
-            return result;
-        }
-    }
-    /**
-     * Get current migration progress
-     */
-    getProgress() {
-        return { ...this.progress };
-    }
-    // ===== Source Loaders =====
-    async loadFromSource() {
-        switch (this.config.source) {
-            case 'sqlite':
-                return this.loadFromSQLite();
-            case 'markdown':
-                return this.loadFromMarkdown();
-            case 'json':
-                return this.loadFromJSON();
-            case 'memory-manager':
-                return this.loadFromMemoryManager();
-            case 'swarm-memory':
-                return this.loadFromSwarmMemory();
-            case 'distributed-memory':
-                return this.loadFromDistributedMemory();
-            default:
-                throw new Error(`Unknown migration source: ${this.config.source}`);
-        }
-    }
-    async loadFromSQLite() {
-        const entries = [];
-        const dbPath = this.config.sourcePath;
-        try {
-            // Dynamic import for better-sqlite3 or similar
-            // In production, would use actual SQLite library
-            const fileContent = await fs.readFile(dbPath);
-            // Parse SQLite format (simplified - actual implementation would use SQLite library)
-            // For now, we'll try to read it as a JSON export format
-            if (dbPath.endsWith('.json')) {
-                const data = JSON.parse(fileContent.toString());
-                if (Array.isArray(data)) {
-                    return data;
-                }
-                else if (data.entries) {
-                    return data.entries;
-                }
-            }
-            // SQLite parsing would go here using better-sqlite3 or sql.js
-            this.emit('migration:warning', {
-                message: 'Direct SQLite parsing requires additional setup. Using export format.',
-            });
-            return entries;
-        }
-        catch (error) {
-            throw new Error(`Failed to load SQLite: ${error.message}`);
-        }
-    }
-    async loadFromMarkdown() {
-        const entries = [];
-        const basePath = this.config.sourcePath;
-        try {
-            const files = await this.walkDirectory(basePath, '.md');
-            for (const filePath of files) {
-                try {
-                    const content = await fs.readFile(filePath, 'utf-8');
-                    const entry = this.parseMarkdownEntry(filePath, content, basePath);
-                    if (entry) {
-                        entries.push(entry);
-                    }
-                }
-                catch (error) {
-                    this.addError(filePath, error.message, 'PARSE_ERROR', true);
-                }
-            }
-            return entries;
-        }
-        catch (error) {
-            throw new Error(`Failed to load Markdown: ${error.message}`);
-        }
-    }
-    async loadFromJSON() {
-        const filePath = this.config.sourcePath;
-        try {
-            const content = await fs.readFile(filePath, 'utf-8');
-            const data = JSON.parse(content);
-            // Handle different JSON formats
-            if (Array.isArray(data)) {
-                return data;
-            }
-            else if (data.entries) {
-                return data.entries;
-            }
-            else if (typeof data === 'object') {
-                // Assume it's a namespace -> entries map
-                const entries = [];
-                for (const [namespace, namespaceEntries] of Object.entries(data)) {
-                    if (Array.isArray(namespaceEntries)) {
-                        for (const entry of namespaceEntries) {
-                            entries.push({ ...entry, namespace });
-                        }
-                    }
-                }
-                return entries;
-            }
-            return [];
-        }
-        catch (error) {
-            throw new Error(`Failed to load JSON: ${error.message}`);
-        }
-    }
-    async loadFromMemoryManager() {
-        // Would integrate with existing MemoryManager instance
-        // For now, try to load from common paths
-        const possiblePaths = [
-            './memory/memory-store.json',
-            './.swarm/memory.db',
-            './memory.json',
-        ];
-        for (const p of possiblePaths) {
-            try {
-                const fullPath = path.resolve(this.config.sourcePath, p);
-                await fs.access(fullPath);
-                return this.loadFromJSON();
-            }
-            catch {
-                continue;
-            }
-        }
-        return [];
-    }
-    async loadFromSwarmMemory() {
-        // Would integrate with SwarmMemory partitions
-        const entries = [];
-        const basePath = this.config.sourcePath;
-        try {
-            // Check for swarm memory directory structure
-            const partitionsPath = path.join(basePath, '.swarm', 'memory');
-            const files = await this.walkDirectory(partitionsPath, '.json');
-            for (const filePath of files) {
-                try {
-                    const content = await fs.readFile(filePath, 'utf-8');
-                    const data = JSON.parse(content);
-                    // Extract namespace from file path
-                    const relativePath = path.relative(partitionsPath, filePath);
-                    const namespace = path.dirname(relativePath).replace(/\\/g, '/');
-                    if (Array.isArray(data)) {
-                        entries.push(...data.map((e) => ({ ...e, namespace })));
-                    }
-                    else if (data.entries) {
-                        entries.push(...data.entries.map((e) => ({ ...e, namespace })));
-                    }
-                }
-                catch (error) {
-                    this.addError(filePath, error.message, 'PARSE_ERROR', true);
-                }
-            }
-            return entries;
-        }
-        catch (error) {
-            return [];
-        }
-    }
-    async loadFromDistributedMemory() {
-        // Would integrate with DistributedMemorySystem nodes
-        return this.loadFromSwarmMemory(); // Similar structure
-    }
-    // ===== Batch Processing =====
-    async processBatch(batch) {
-        for (const legacyEntry of batch) {
-            try {
-                // Validate if enabled
-                if (this.config.validateData) {
-                    const validation = this.validateEntry(legacyEntry);
-                    if (!validation.valid) {
-                        if (this.config.continueOnError) {
-                            this.addError(legacyEntry.key || 'unknown', validation.reason || 'Validation failed', 'VALIDATION_ERROR', false);
-                            this.progress.skipped++;
-                            continue;
-                        }
-                        else {
-                            throw new Error(validation.reason);
-                        }
-                    }
-                }
-                // Transform to new format
-                const newEntry = await this.transformEntry(legacyEntry);
-                // Store in target
-                await this.target.store(newEntry);
-                this.progress.migrated++;
-            }
-            catch (error) {
-                if (this.config.continueOnError) {
-                    this.addError(legacyEntry.key || 'unknown', error.message, 'STORE_ERROR', true);
-                    this.progress.failed++;
-                }
-                else {
-                    throw error;
-                }
-            }
-        }
-    }
-    async transformEntry(legacy) {
-        // Map namespace if configured
-        let namespace = legacy.namespace || 'default';
-        if (this.config.namespaceMapping && this.config.namespaceMapping[namespace]) {
-            namespace = this.config.namespaceMapping[namespace];
-        }
-        // Determine content
-        const content = typeof legacy.value === 'string'
-            ? legacy.value
-            : JSON.stringify(legacy.value);
-        // Map type if configured
-        let type = 'semantic';
-        if (legacy.metadata?.type && typeof legacy.metadata.type === 'string') {
-            if (this.config.typeMapping && this.config.typeMapping[legacy.metadata.type]) {
-                type = this.config.typeMapping[legacy.metadata.type];
-            }
-            else if (this.isValidMemoryType(legacy.metadata.type)) {
-                type = legacy.metadata.type;
-            }
-        }
-        // Parse timestamps
-        const createdAt = this.parseTimestamp(legacy.createdAt || legacy.created_at || legacy.timestamp);
-        const updatedAt = this.parseTimestamp(legacy.updatedAt || legacy.updated_at || legacy.timestamp);
-        const input = {
-            key: legacy.key,
-            content,
-            type,
-            namespace,
-            tags: legacy.tags || [],
-            metadata: {
-                ...legacy.metadata,
-                migrated: true,
-                migrationSource: this.config.source,
-                migrationTimestamp: Date.now(),
-                originalValue: legacy.value,
-            },
-        };
-        const entry = createDefaultEntry(input);
-        entry.createdAt = createdAt;
-        entry.updatedAt = updatedAt;
-        // Generate embedding if configured
-        if (this.config.generateEmbeddings && this.embeddingGenerator) {
-            try {
-                entry.embedding = await this.embeddingGenerator(content);
-            }
-            catch (error) {
-                // Log but don't fail
-                this.emit('migration:warning', {
-                    message: `Failed to generate embedding for ${legacy.key}: ${error.message}`,
-                });
-            }
-        }
-        return entry;
-    }
-    // ===== Helper Methods =====
-    initializeProgress() {
-        return {
-            total: 0,
-            migrated: 0,
-            failed: 0,
-            skipped: 0,
-            currentBatch: 0,
-            totalBatches: 0,
-            percentage: 0,
-            estimatedTimeRemaining: 0,
-            errors: [],
-        };
-    }
-    validateEntry(entry) {
-        if (!entry.key || typeof entry.key !== 'string') {
-            return { valid: false, reason: 'Missing or invalid key' };
-        }
-        if (entry.value === undefined) {
-            return { valid: false, reason: 'Missing value' };
-        }
-        if (entry.key.length > 500) {
-            return { valid: false, reason: 'Key too long (max 500 chars)' };
-        }
-        return { valid: true };
-    }
-    addError(entryId, message, code, recoverable) {
-        const error = {
-            entryId,
-            message,
-            code,
-            recoverable,
-        };
-        this.progress.errors.push(error);
-        this.emit('migration:error', error);
-    }
-    parseTimestamp(value) {
-        if (!value)
-            return Date.now();
-        if (typeof value === 'number') {
-            // Handle both milliseconds and seconds
-            return value > 1e12 ? value : value * 1000;
-        }
-        const parsed = Date.parse(value);
-        return isNaN(parsed) ? Date.now() : parsed;
-    }
-    isValidMemoryType(type) {
-        return ['episodic', 'semantic', 'procedural', 'working', 'cache'].includes(type);
-    }
-    estimateTimeRemaining(startTime, completed, total) {
-        if (completed === 0)
-            return 0;
-        const elapsed = Date.now() - startTime;
-        const rate = completed / elapsed;
-        const remaining = total - completed;
-        return Math.round(remaining / rate);
-    }
-    generateSummary() {
-        const { migrated, failed, skipped, total, errors } = this.progress;
-        let summary = `Migrated ${migrated}/${total} entries`;
-        if (failed > 0) {
-            summary += `, ${failed} failed`;
-        }
-        if (skipped > 0) {
-            summary += `, ${skipped} skipped`;
-        }
-        if (errors.length > 0) {
-            const errorTypes = new Map();
-            for (const error of errors) {
-                errorTypes.set(error.code, (errorTypes.get(error.code) || 0) + 1);
-            }
-            const errorSummary = Array.from(errorTypes.entries())
-                .map(([code, count]) => `${code}: ${count}`)
-                .join(', ');
-            summary += `. Errors: ${errorSummary}`;
-        }
-        return summary;
-    }
-    async walkDirectory(dir, extension) {
-        const files = [];
-        try {
-            const entries = await fs.readdir(dir, { withFileTypes: true });
-            for (const entry of entries) {
-                const fullPath = path.join(dir, entry.name);
-                if (entry.isDirectory()) {
-                    const subFiles = await this.walkDirectory(fullPath, extension);
-                    files.push(...subFiles);
-                }
-                else if (entry.isFile() && entry.name.endsWith(extension)) {
-                    files.push(fullPath);
-                }
-            }
-        }
-        catch (error) {
-            // Directory doesn't exist or isn't readable
-        }
-        return files;
-    }
-    parseMarkdownEntry(filePath, content, basePath) {
-        // Extract frontmatter if present
-        const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
-        let metadata = {};
-        let body = content;
-        if (frontmatterMatch) {
-            try {
-                // Simple YAML-like parsing
-                const frontmatter = frontmatterMatch[1];
-                for (const line of frontmatter.split('\n')) {
-                    const colonIndex = line.indexOf(':');
-                    if (colonIndex > 0) {
-                        const key = line.substring(0, colonIndex).trim();
-                        let value = line.substring(colonIndex + 1).trim();
-                        // Parse common types
-                        if (value === 'true')
-                            value = true;
-                        else if (value === 'false')
-                            value = false;
-                        else if (typeof value === 'string' && /^\d+$/.test(value))
-                            value = parseInt(value, 10);
-                        else if (typeof value === 'string' && value.startsWith('[') && value.endsWith(']')) {
-                            try {
-                                value = JSON.parse(value.replace(/'/g, '"'));
-                            }
-                            catch {
-                                // Keep as string
-                            }
-                        }
-                        metadata[key] = value;
-                    }
-                }
-                body = frontmatterMatch[2];
-            }
-            catch {
-                // Failed to parse frontmatter, use whole content
-            }
-        }
-        // Derive key from file path
-        const relativePath = path.relative(basePath, filePath);
-        const key = relativePath
-            .replace(/\\/g, '/')
-            .replace(/\.md$/, '')
-            .replace(/\//g, ':');
-        // Derive namespace from directory structure
-        const namespace = path.dirname(relativePath).replace(/\\/g, '/') || 'default';
-        return {
-            key,
-            value: body.trim(),
-            namespace,
-            tags: Array.isArray(metadata.tags) ? metadata.tags : [],
-            metadata,
-            timestamp: Date.now(),
-        };
-    }
-}
-/**
- * Convenience function to create a migrator
- */
-export function createMigrator(target, source, sourcePath, options = {}, embeddingGenerator) {
-    return new MemoryMigrator(target, { source, sourcePath, ...options }, embeddingGenerator);
-}
-/**
- * Migrate from multiple sources
- */
-export async function migrateMultipleSources(target, sources, options = {}, embeddingGenerator) {
-    const results = [];
-    for (const { source, path: sourcePath } of sources) {
-        const migrator = createMigrator(target, source, sourcePath, options, embeddingGenerator);
-        const result = await migrator.migrate();
-        results.push(result);
-    }
-    return results;
-}
-export default MemoryMigrator;
-//# sourceMappingURL=migration.js.map

package/src/@claude-flow/memory/dist/persistent-sona.d.ts

@@ -1,144 +0,0 @@
-/**
- * PersistentSonaCoordinator - SONA learning with RVF persistence
- *
- * Wraps RvfLearningStore to provide an in-memory pattern bank with
- * brute-force cosine similarity, trajectory buffering, EWC tracking,
- * and automatic periodic persistence to disk.
- *
- * This is intentionally decoupled from the ruvector SONA classes:
- * it defines its own compatible types and delegates persistence to
- * RvfLearningStore.
- *
- * @module @claude-flow/memory/persistent-sona
- */
-import type { PatternRecord, TrajectoryRecord } from './rvf-learning-store.js';
-export interface PersistentSonaConfig {
-    /** Path to the RVF learning store file */
-    storePath: string;
-    /** Cosine similarity threshold for pattern matching (default: 0.85) */
-    patternThreshold?: number;
-    /** Maximum buffered trajectories before oldest are evicted (default: 1000) */
-    maxTrajectoryBuffer?: number;
-    /** Auto-persist interval in ms (default: 30000) */
-    autoPersistInterval?: number;
-    /** Enable verbose logging (default: false) */
-    verbose?: boolean;
-}
-/**
- * Coordinates SONA learning with persistent storage.
- *
- * @example
- * ```typescript
- * const sona = new PersistentSonaCoordinator({
- *   storePath: './data/sona-learning.rvls',
- * });
- * await sona.initialize();
- *
- * // Store a pattern
- * const id = sona.storePattern('query_response', embedding);
- *
- * // Find similar patterns
- * const matches = sona.findSimilarPatterns(queryEmbedding, 5);
- *
- * // Record a trajectory
- * sona.recordTrajectory({ id: 'traj-1', steps: [...], outcome: 'success', ... });
- *
- * // Periodic background learning
- * const result = sona.runBackgroundLoop();
- *
- * await sona.shutdown();
- * ```
- */
-export declare class PersistentSonaCoordinator {
-    private store;
-    private patterns;
-    private trajectoryBuffer;
-    private ewcState;
-    private patternThreshold;
-    private maxTrajectoryBuffer;
-    private verbose;
-    private initialized;
-    constructor(config: PersistentSonaConfig);
-    /**
-     * Initialize by loading persisted state from the RVF store.
-     */
-    initialize(): Promise<void>;
-    /**
-     * Store a new pattern and return its ID.
-     *
-     * @param type - Pattern type (e.g. 'query_response', 'routing')
-     * @param embedding - The pattern embedding vector
-     * @param metadata - Optional extra metadata (currently unused, reserved)
-     * @returns The generated pattern ID
-     */
-    storePattern(type: string, embedding: number[], metadata?: Record<string, unknown>): string;
-    /**
-     * Find the k most similar patterns above the configured threshold.
-     * Uses brute-force cosine similarity (suitable for small pattern sets).
-     */
-    findSimilarPatterns(embedding: number[], k?: number): PatternRecord[];
-    /**
-     * Record a pattern usage outcome. Updates the success rate using an
-     * exponential moving average (alpha = 0.1).
-     */
-    recordPatternUsage(patternId: string, success: boolean): void;
-    /**
-     * Remove patterns that have low success rates after sufficient usage.
-     *
-     * @returns The number of patterns pruned
-     */
-    prunePatterns(minSuccessRate?: number, minUseCount?: number): number;
-    /**
-     * Buffer a completed trajectory for later processing.
-     * When the buffer exceeds maxTrajectoryBuffer, the oldest entries
-     * are evicted.
-     */
-    recordTrajectory(trajectory: TrajectoryRecord): void;
-    /**
-     * Process buffered trajectories to extract new patterns.
-     * Successful and partial trajectories are mined for high-confidence
-     * steps; new patterns are stored if they are sufficiently different
-     * from existing ones.
-     *
-     * After processing, the trajectory buffer is cleared and low-performing
-     * patterns are pruned.
-     *
-     * @returns Summary of the learning pass
-     */
-    runBackgroundLoop(): {
-        patternsLearned: number;
-        trajectoriesProcessed: number;
-    };
-    /**
-     * Flush current in-memory state to the RVF store on disk.
-     */
-    persist(): Promise<void>;
-    /**
-     * Persist state and shut down the store.
-     */
-    shutdown(): Promise<void>;
-    /**
-     * Return a summary of the coordinator's current state.
-     */
-    getStats(): {
-        patterns: number;
-        avgSuccessRate: number;
-        trajectoriesBuffered: number;
-        ewcTasksLearned: number;
-    };
-    /**
-     * Extract patterns from a trajectory's high-confidence steps.
-     * A step produces a new pattern only if no sufficiently similar
-     * pattern already exists.
-     */
-    private extractPatternsFromTrajectory;
-    /**
-     * Deterministic hash-based embedding for pattern extraction.
-     * This is a lightweight stand-in for a real embedding model,
-     * matching the approach used in SonaCoordinator.
-     */
-    private createHashEmbedding;
-    private ensureInitialized;
-    private log;
-}
-//# sourceMappingURL=persistent-sona.d.ts.map