agentic-qe 3.7.21 → 3.8.0

package/dist/learning/v2-to-v3-migration.js

@@ -1,529 +0,0 @@
-/**
- * V2 to V3 Data Migration Module
- * ADR-038: V3 QE Memory Unification
- *
- * Migrates data from V2 AQE fleet (.agentic-qe/memory.db) to V3 systems:
- * - patterns → RealQEReasoningBank (.agentic-qe/memory.db)
- * - captured_experiences → QEUnifiedMemory learning domain
- * - learning_experiences → RL training data
- * - concept_nodes/edges → Code intelligence knowledge graph
- *
- * @module migration
- */
-import { openDatabase } from '../shared/safe-db.js';
-import { toErrorMessage } from '../shared/error-utils.js';
-import { safeJsonParse } from '../shared/safe-json.js';
-import { LoggerFactory } from '../logging/index.js';
-const logger = LoggerFactory.create('v2-to-v3-migration');
-// ============================================================================
-// V2 to V3 Migration Class
-// ============================================================================
-export class V2ToV3Migrator {
-    config;
-    v2Db = null;
-    v3Db = null;
-    startTime = 0;
-    constructor(config) {
-        this.config = config;
-    }
-    /**
-     * Execute the full migration
-     */
-    async migrate() {
-        this.startTime = Date.now();
-        const errors = [];
-        const tablesMigrated = [];
-        const counts = {};
-        try {
-            // Step 1: Connect to databases
-            await this.reportProgress('connecting', 0, 1, 'Connecting to databases...');
-            await this.connect();
-            // Step 2: Read V2 data
-            await this.reportProgress('reading', 0, 1, 'Reading V2 data...');
-            const v2Data = await this.readV2Data();
-            // Step 3: Migrate to V3
-            await this.reportProgress('migrating', 0, 1, 'Migrating patterns...');
-            // Migrate patterns → RealQEReasoningBank
-            const patternCount = await this.migratePatterns(v2Data.patterns);
-            if (patternCount > 0) {
-                tablesMigrated.push('patterns');
-                counts.patterns = patternCount;
-            }
-            await this.reportProgress('migrating', 1, 4, 'Migrating captured experiences...');
-            // Migrate captured experiences → learning domain
-            const experienceCount = await this.migrateCapturedExperiences(v2Data.capturedExperiences);
-            if (experienceCount > 0) {
-                tablesMigrated.push('captured_experiences');
-                counts.captured_experiences = experienceCount;
-            }
-            await this.reportProgress('migrating', 2, 4, 'Migrating learning experiences...');
-            // Migrate learning experiences → RL data
-            const learningCount = await this.migrateLearningExperiences(v2Data.learningExperiences);
-            if (learningCount > 0) {
-                tablesMigrated.push('learning_experiences');
-                counts.learning_experiences = learningCount;
-            }
-            await this.reportProgress('migrating', 3, 4, 'Migrating concept graph...');
-            // Migrate concept nodes/edges → code intelligence
-            const conceptCount = await this.migrateConceptGraph(v2Data.conceptNodes, v2Data.conceptEdges);
-            if (conceptCount > 0) {
-                tablesMigrated.push('concept_graph');
-                counts.concept_graph = conceptCount;
-            }
-            // Step 4: Validate migration
-            await this.reportProgress('validating', 0, 1, 'Validating migration...');
-            const validationErrors = await this.validateMigration();
-            errors.push(...validationErrors);
-            await this.reportProgress('complete', 1, 1, 'Migration complete!');
-            return {
-                success: errors.length === 0,
-                tablesMigrated,
-                counts,
-                errors,
-                duration: Date.now() - this.startTime,
-            };
-        }
-        catch (error) {
-            errors.push(toErrorMessage(error));
-            return {
-                success: false,
-                tablesMigrated,
-                counts,
-                errors,
-                duration: Date.now() - this.startTime,
-            };
-        }
-        finally {
-            await this.disconnect();
-        }
-    }
-    // -------------------------------------------------------------------------
-    // Private Methods
-    // -------------------------------------------------------------------------
-    /**
-     * Safely parse JSON, returning a wrapper object for non-JSON strings
-     * Handles V2 data where state/action columns may contain plain strings like "task-started"
-     */
-    safeJsonParse(value, fieldName = 'value') {
-        if (value === null || value === undefined || value === '') {
-            return {};
-        }
-        // Trim whitespace
-        const trimmed = value.trim();
-        // Check if it looks like JSON (starts with { or [)
-        if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
-            try {
-                return safeJsonParse(trimmed);
-            }
-            catch (e) {
-                // If parsing fails, wrap as string
-                logger.debug('JSON parse failed during migration', { field: fieldName, error: e instanceof Error ? e.message : String(e) });
-                return { [fieldName]: trimmed, _parseError: true };
-            }
-        }
-        // Not JSON - wrap the plain string value
-        return { [fieldName]: trimmed, _isPlainString: true };
-    }
-    async reportProgress(stage, current, total, message, table) {
-        if (this.config.onProgress) {
-            this.config.onProgress({ stage, table, current, total, message });
-        }
-    }
-    async connect() {
-        // Connect to V2 database (readonly)
-        this.v2Db = openDatabase(this.config.v2DbPath, { readonly: true });
-        // Skip WAL mode for readonly database
-        // Connect to V3 patterns database (readwrite, openDatabase sets WAL + busy_timeout)
-        this.v3Db = openDatabase(this.config.v3PatternsDbPath);
-        // Ensure V3 schema exists
-        this.createV3Schema();
-    }
-    async disconnect() {
-        if (this.v2Db) {
-            this.v2Db.close();
-            this.v2Db = null;
-        }
-        if (this.v3Db) {
-            this.v3Db.close();
-            this.v3Db = null;
-        }
-    }
-    createV3Schema() {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        // This schema should match SQLitePatternStore's schema
-        this.v3Db.exec(`
-      CREATE TABLE IF NOT EXISTS qe_patterns (
-        id TEXT PRIMARY KEY,
-        pattern_type TEXT NOT NULL,
-        qe_domain TEXT NOT NULL,
-        domain TEXT NOT NULL,
-        name TEXT NOT NULL,
-        description TEXT,
-        confidence REAL DEFAULT 0.5,
-        usage_count INTEGER DEFAULT 0,
-        success_rate REAL DEFAULT 0.0,
-        quality_score REAL DEFAULT 0.0,
-        tier TEXT DEFAULT 'short-term',
-        template_json TEXT,
-        context_json TEXT,
-        created_at TEXT DEFAULT (datetime('now')),
-        updated_at TEXT DEFAULT (datetime('now')),
-        last_used_at TEXT,
-        successful_uses INTEGER DEFAULT 0
-      );
-
-      CREATE TABLE IF NOT EXISTS qe_pattern_embeddings (
-        pattern_id TEXT PRIMARY KEY,
-        embedding BLOB NOT NULL,
-        dimension INTEGER NOT NULL,
-        model TEXT DEFAULT 'all-MiniLM-L6-v2',
-        created_at TEXT DEFAULT (datetime('now')),
-        FOREIGN KEY (pattern_id) REFERENCES qe_patterns(id) ON DELETE CASCADE
-      );
-
-      CREATE TABLE IF NOT EXISTS qe_pattern_usage (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        pattern_id TEXT NOT NULL,
-        success INTEGER NOT NULL,
-        metrics_json TEXT,
-        feedback TEXT,
-        created_at TEXT DEFAULT (datetime('now'))
-      );
-
-      CREATE INDEX IF NOT EXISTS idx_patterns_domain ON qe_patterns(qe_domain);
-      CREATE INDEX IF NOT EXISTS idx_patterns_type ON qe_patterns(pattern_type);
-      CREATE INDEX IF NOT EXISTS idx_patterns_tier ON qe_patterns(tier);
-    `);
-    }
-    async readV2Data() {
-        if (!this.v2Db)
-            throw new Error('V2 database not connected');
-        // Helper to safely read from a table that might not exist
-        const safeReadTable = (tableName) => {
-            try {
-                // Check if table exists first
-                const tableExists = this.v2Db.prepare(`
-          SELECT name FROM sqlite_master
-          WHERE type='table' AND name=?
-        `).get(tableName);
-                if (!tableExists) {
-                    console.log(`  [V2Migration] Table '${tableName}' not found, skipping...`);
-                    return [];
-                }
-                return this.v2Db.prepare(`SELECT * FROM ${tableName}`).all();
-            }
-            catch (error) {
-                console.warn(`  [V2Migration] Could not read table '${tableName}': ${toErrorMessage(error)}`);
-                return [];
-            }
-        };
-        return {
-            patterns: safeReadTable('patterns'),
-            capturedExperiences: safeReadTable('captured_experiences'),
-            learningExperiences: safeReadTable('learning_experiences'),
-            conceptNodes: safeReadTable('concept_nodes'),
-            conceptEdges: safeReadTable('concept_edges'),
-        };
-    }
-    /**
-     * Migrate V2 patterns to V3 qe_patterns table
-     */
-    async migratePatterns(patterns) {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        if (patterns.length === 0)
-            return 0;
-        const insert = this.v3Db.prepare(`
-      INSERT OR REPLACE INTO qe_patterns (
-        id, pattern_type, qe_domain, domain, name, description,
-        confidence, usage_count, success_rate, tier, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        const insertEmbedding = this.v3Db.prepare(`
-      INSERT OR REPLACE INTO qe_pattern_embeddings (pattern_id, embedding, dimension, model)
-      VALUES (?, ?, ?, ?)
-    `);
-        let count = 0;
-        const transaction = this.v3Db.transaction(() => {
-            for (const pattern of patterns) {
-                try {
-                    // Parse V2 pattern data - use safe parsing for non-JSON values
-                    const patternData = this.safeJsonParse(pattern.pattern, 'pattern');
-                    const metadata = this.safeJsonParse(pattern.metadata, 'metadata');
-                    // Map V2 domain to V3 QEDomain
-                    const qeDomain = this.mapV2DomainToV3(pattern.domain || String(metadata.domain || 'general'));
-                    // Determine pattern type from metadata or pattern content
-                    const patternType = this.inferPatternType(patternData, metadata);
-                    // Calculate quality score from confidence and success rate
-                    const qualityScore = (pattern.confidence * 0.6) + (pattern.success_rate * 0.4);
-                    // Get pattern name safely
-                    const patternName = String(patternData.name || metadata.name || pattern.pattern.substring(0, 50));
-                    // Insert pattern
-                    insert.run(pattern.id, patternType, qeDomain, pattern.domain || 'general', patternName, String(patternData.description || metadata.description || '') || null, pattern.confidence, pattern.usage_count, pattern.success_rate, this.calculateTier(pattern.usage_count, qualityScore), new Date(pattern.created_at * 1000).toISOString());
-                    // TODO: Migrate embeddings if available (V2 doesn't store them in patterns table)
-                    // Would need to generate embeddings or migrate from captured_experiences
-                    count++;
-                }
-                catch (error) {
-                    console.warn(`[V2Migration] Failed to migrate pattern ${pattern.id}:`, error);
-                }
-            }
-        });
-        transaction();
-        return count;
-    }
-    /**
-     * Migrate captured experiences to learning domain
-     * These are stored in memory_entries for now, could be moved to QEUnifiedMemory
-     */
-    async migrateCapturedExperiences(experiences) {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        if (experiences.length === 0)
-            return 0;
-        // Store captured experiences as learning patterns
-        const insert = this.v3Db.prepare(`
-      INSERT OR REPLACE INTO qe_patterns (
-        id, pattern_type, qe_domain, domain, name, description,
-        confidence, usage_count, success_rate, tier, context_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        let count = 0;
-        for (const exp of experiences) {
-            try {
-                // Use safe parsing - V2 may store non-JSON values
-                const execution = this.safeJsonParse(exp.execution, 'execution');
-                const context = this.safeJsonParse(exp.context, 'context');
-                const outcome = this.safeJsonParse(exp.outcome, 'outcome');
-                const qeDomain = this.mapTaskTypeToDomain(exp.task_type);
-                // Safely access outcome.success with fallback
-                const isSuccess = typeof outcome.success === 'boolean' ? outcome.success : false;
-                insert.run(exp.id, // Use original ID
-                'test-template', // Captured experiences become templates
-                qeDomain, exp.task_type, `Experience: ${exp.task_type}`, `Agent: ${exp.agent_type}, Task: ${exp.task_type}`, isSuccess ? 0.8 : 0.3, // Confidence based on outcome
-                1, // One usage
-                isSuccess ? 1.0 : 0.0, // Success rate
-                'short-term', JSON.stringify({ context, execution, outcome }), new Date(exp.created_at * 1000).toISOString());
-                count++;
-            }
-            catch (error) {
-                console.warn(`[V2Migration] Failed to migrate experience ${exp.id}:`, error);
-            }
-        }
-        return count;
-    }
-    /**
-     * Migrate learning experiences (RL data)
-     * Store as learning patterns for SONA integration
-     */
-    async migrateLearningExperiences(experiences) {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        if (experiences.length === 0)
-            return 0;
-        const insert = this.v3Db.prepare(`
-      INSERT OR REPLACE INTO qe_patterns (
-        id, pattern_type, qe_domain, domain, name, description,
-        confidence, usage_count, success_rate, tier, context_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        let count = 0;
-        for (const exp of experiences) {
-            try {
-                // Use safe parsing - V2 may store plain strings like "task-started" instead of JSON
-                const state = this.safeJsonParse(exp.state, 'state');
-                const nextState = exp.next_state ? this.safeJsonParse(exp.next_state, 'nextState') : null;
-                const metadata = this.safeJsonParse(exp.metadata, 'metadata');
-                const qeDomain = this.mapTaskTypeToDomain(exp.task_type);
-                insert.run(`le_${exp.id}`, // Prefix to avoid ID collision
-                'test-template', qeDomain, exp.task_type, `RL Experience: ${exp.task_type}`, `Action: ${exp.action}, Reward: ${exp.reward.toFixed(2)}`, Math.min(1.0, Math.max(0.1, exp.reward)), // Confidence from reward
-                1, exp.reward > 0 ? 1.0 : 0.0, 'short-term', JSON.stringify({ state, action: exp.action, reward: exp.reward, nextState, metadata }), new Date(exp.created_at).toISOString());
-                count++;
-            }
-            catch (error) {
-                console.warn(`[V2Migration] Failed to migrate learning experience ${exp.id}:`, error);
-            }
-        }
-        return count;
-    }
-    /**
-     * Migrate concept graph (nodes and edges) for code intelligence
-     */
-    async migrateConceptGraph(nodes, edges) {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        if (nodes.length === 0)
-            return 0;
-        const insert = this.v3Db.prepare(`
-      INSERT OR REPLACE INTO qe_patterns (
-        id, pattern_type, qe_domain, domain, name, description,
-        confidence, usage_count, success_rate, tier, context_json, created_at
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        let count = 0;
-        // Store concept nodes as knowledge graph patterns
-        for (const node of nodes) {
-            try {
-                // Use safe parsing for non-JSON values
-                const properties = this.safeJsonParse(node.properties, 'properties');
-                const embedding = this.safeJsonParse(node.embedding, 'embedding');
-                insert.run(`cn_${node.id}`, // Prefix for concept node
-                'test-template', 'code-intelligence', node.type, node.name, String(properties.description || '') || `Concept: ${node.name}`, node.activation_level || 0.5, 0, 1.0, // Concepts are valid by default
-                'medium-term', JSON.stringify({
-                    type: node.type,
-                    domain: node.domain,
-                    properties,
-                    embedding,
-                    activationLevel: node.activation_level,
-                }), new Date(node.created_at * 1000).toISOString());
-                count++;
-            }
-            catch (error) {
-                console.warn(`[V2Migration] Failed to migrate concept node ${node.id}:`, error);
-            }
-        }
-        // Store edges as relationship patterns
-        for (const edge of edges) {
-            try {
-                insert.run(`ce_${edge.id}`, // Prefix for concept edge
-                'test-template', 'code-intelligence', edge.type, `${edge.source} → ${edge.target}`, `Weight: ${edge.weight.toFixed(2)}, Evidence: ${edge.evidence}`, edge.weight, 0, 1.0, 'short-term', JSON.stringify({
-                    source: edge.source,
-                    target: edge.target,
-                    weight: edge.weight,
-                    evidence: edge.evidence,
-                }), new Date(edge.created_at * 1000).toISOString());
-                count++;
-            }
-            catch (error) {
-                console.warn(`[V2Migration] Failed to migrate concept edge ${edge.id}:`, error);
-            }
-        }
-        return count;
-    }
-    /**
-     * Validate migration results
-     */
-    async validateMigration() {
-        if (!this.v3Db)
-            throw new Error('V3 database not connected');
-        const errors = [];
-        // Check pattern counts (0 is valid if v2 had no patterns)
-        const patternCount = this.v3Db.prepare('SELECT COUNT(*) as count FROM qe_patterns').get();
-        if (patternCount.count === 0) {
-            // Not an error - v2 might have had no patterns to migrate
-            console.log('  [V2Migration] Note: No patterns were migrated (v2 database may have been empty)');
-        }
-        // Check for duplicate IDs (actual error)
-        const duplicates = this.v3Db.prepare(`
-      SELECT id, COUNT(*) as count FROM qe_patterns GROUP BY id HAVING count > 1
-    `).all();
-        if (duplicates.length > 0) {
-            errors.push(`${duplicates.length} duplicate pattern IDs found`);
-        }
-        return errors;
-    }
-    // -------------------------------------------------------------------------
-    // Mapping Helpers
-    // -------------------------------------------------------------------------
-    mapV2DomainToV3(v2Domain) {
-        const domainMap = {
-            'test-generation': 'test-generation',
-            'test': 'test-generation',
-            'testing': 'test-generation',
-            'coverage': 'coverage-analysis',
-            'coverage-analysis': 'coverage-analysis',
-            'quality': 'quality-assessment',
-            'defect': 'defect-intelligence',
-            'security': 'security-compliance',
-            'code-intelligence': 'code-intelligence',
-            'knowledge': 'code-intelligence',
-            'learning': 'learning-optimization',
-            'contract': 'contract-testing',
-            'visual': 'visual-accessibility',
-            'a11y': 'visual-accessibility',
-            'chaos': 'chaos-resilience',
-            'performance': 'chaos-resilience',
-            'requirements': 'requirements-validation',
-            'general': 'test-generation', // Default to test-generation
-        };
-        return domainMap[v2Domain.toLowerCase()] || 'test-generation';
-    }
-    mapTaskTypeToDomain(taskType) {
-        const typeLower = taskType.toLowerCase();
-        if (typeLower.includes('test') || typeLower.includes('spec'))
-            return 'test-generation';
-        if (typeLower.includes('coverage'))
-            return 'coverage-analysis';
-        if (typeLower.includes('defect') || typeLower.includes('bug'))
-            return 'defect-intelligence';
-        if (typeLower.includes('security') || typeLower.includes('vuln'))
-            return 'security-compliance';
-        if (typeLower.includes('concept') || typeLower.includes('semantic'))
-            return 'code-intelligence';
-        if (typeLower.includes('learn') || typeLower.includes('pattern'))
-            return 'learning-optimization';
-        if (typeLower.includes('contract') || typeLower.includes('api'))
-            return 'contract-testing';
-        if (typeLower.includes('visual') || typeLower.includes('a11y'))
-            return 'visual-accessibility';
-        if (typeLower.includes('chaos') || typeLower.includes('performance'))
-            return 'chaos-resilience';
-        return 'test-generation'; // Default
-    }
-    inferPatternType(patternData, metadata) {
-        const data = patternData;
-        const meta = metadata || {};
-        // Check metadata first
-        if (meta.patternType)
-            return String(meta.patternType);
-        if (data.type)
-            return String(data.type);
-        // Infer from pattern content
-        const patternStr = JSON.stringify(patternData).toLowerCase();
-        const metaStr = JSON.stringify(metadata).toLowerCase();
-        if (patternStr.includes('mock') || metaStr.includes('mock'))
-            return 'mock-pattern';
-        if (patternStr.includes('assert') || metaStr.includes('assert'))
-            return 'assertion-pattern';
-        if (patternStr.includes('coverage'))
-            return 'coverage-strategy';
-        if (patternStr.includes('mutation'))
-            return 'mutation-strategy';
-        if (patternStr.includes('contract') || patternStr.includes('api'))
-            return 'api-contract';
-        if (patternStr.includes('visual') || patternStr.includes('snapshot'))
-            return 'visual-baseline';
-        if (patternStr.includes('a11y') || patternStr.includes('accessibility'))
-            return 'a11y-check';
-        if (patternStr.includes('performance') || patternStr.includes('benchmark'))
-            return 'perf-benchmark';
-        if (patternStr.includes('flaky'))
-            return 'flaky-fix';
-        if (patternStr.includes('refactor'))
-            return 'refactor-safe';
-        return 'test-template'; // Default
-    }
-    calculateTier(usageCount, qualityScore) {
-        if (usageCount >= 10 && qualityScore >= 0.8)
-            return 'long-term';
-        if (usageCount >= 5 || qualityScore >= 0.6)
-            return 'medium-term';
-        return 'short-term';
-    }
-}
-// ============================================================================
-// Factory Function
-// ============================================================================
-/**
- * Create and execute a V2 to V3 migration
- */
-export async function migrateV2ToV3(v2DbPath = '.agentic-qe/memory.db', v3PatternsDbPath = '.agentic-qe/memory.db', onProgress) {
-    const migrator = new V2ToV3Migrator({
-        v2DbPath,
-        v3PatternsDbPath,
-        onProgress,
-    });
-    return migrator.migrate();
-}
-//# sourceMappingURL=v2-to-v3-migration.js.map