claude-flow 2.7.0-alpha.1 → 2.7.0-alpha.11

package/docs/reasoningbank/models/_scripts/schema-validator.cjs

@@ -0,0 +1,360 @@
+#!/usr/bin/env node
+/**
+ * Claude-Flow Schema Validator
+ * Ensures memory.db contains ALL required tables for full claude-flow compatibility
+ */
+
+const Database = require('better-sqlite3');
+const fs = require('fs');
+const path = require('path');
+
+class SchemaValidator {
+  constructor(dbPath) {
+    this.dbPath = dbPath;
+    this.db = new Database(dbPath);
+
+    // Required tables for full claude-flow compatibility
+    this.requiredTables = {
+      // ReasoningBank core tables
+      'patterns': {
+        columns: ['id', 'description', 'context', 'success_rate', 'confidence', 'domain', 'tags', 'created_at', 'updated_at']
+      },
+      'pattern_embeddings': {
+        columns: ['id', 'pattern_id', 'embedding', 'created_at']
+      },
+      'task_trajectories': {
+        columns: ['id', 'memory_id', 'step_number', 'action', 'reasoning', 'outcome', 'created_at']
+      },
+      'pattern_links': {
+        columns: ['id', 'source_id', 'target_id', 'link_type', 'strength', 'created_at']
+      },
+
+      // Claude-Flow memory tables
+      'memories': {
+        columns: ['id', 'key', 'value', 'namespace', 'created_at', 'updated_at', 'expires_at', 'metadata']
+      },
+      'memory_embeddings': {
+        columns: ['id', 'memory_id', 'embedding', 'created_at']
+      },
+
+      // Claude-Flow session tables
+      'sessions': {
+        columns: ['id', 'session_id', 'agent_type', 'started_at', 'ended_at', 'metadata']
+      },
+      'session_metrics': {
+        columns: ['id', 'session_id', 'metric_name', 'metric_value', 'recorded_at']
+      },
+
+      // Claude-Flow neural tables
+      'neural_patterns': {
+        columns: ['id', 'pattern_type', 'pattern_data', 'confidence', 'created_at']
+      },
+      'training_data': {
+        columns: ['id', 'input', 'output', 'success', 'created_at']
+      }
+    };
+  }
+
+  validateSchema() {
+    console.log(`\n🔍 Validating schema for: ${this.dbPath}\n`);
+
+    const results = {
+      valid: true,
+      tables: {},
+      missing: [],
+      incomplete: []
+    };
+
+    // Get existing tables
+    const existingTables = this.db.prepare(`
+      SELECT name FROM sqlite_master WHERE type='table'
+    `).all().map(row => row.name);
+
+    // Check each required table
+    for (const [tableName, schema] of Object.entries(this.requiredTables)) {
+      if (!existingTables.includes(tableName)) {
+        results.missing.push(tableName);
+        results.valid = false;
+        results.tables[tableName] = { exists: false, columns: [] };
+        console.log(`❌ Missing table: ${tableName}`);
+      } else {
+        // Check columns
+        const columns = this.db.prepare(`PRAGMA table_info(${tableName})`).all();
+        const columnNames = columns.map(c => c.name);
+
+        const missingColumns = schema.columns.filter(col => !columnNames.includes(col));
+
+        if (missingColumns.length > 0) {
+          results.incomplete.push({ table: tableName, missingColumns });
+          results.valid = false;
+          console.log(`⚠️  Table ${tableName} missing columns: ${missingColumns.join(', ')}`);
+        } else {
+          console.log(`✅ Table ${tableName} complete`);
+        }
+
+        results.tables[tableName] = {
+          exists: true,
+          columns: columnNames,
+          complete: missingColumns.length === 0
+        };
+      }
+    }
+
+    console.log(`\n${results.valid ? '✅' : '❌'} Schema validation ${results.valid ? 'PASSED' : 'FAILED'}\n`);
+
+    return results;
+  }
+
+  createMissingTables() {
+    console.log('\n🔧 Creating missing tables...\n');
+
+    const results = this.validateSchema();
+
+    // Create missing tables
+    for (const tableName of results.missing) {
+      console.log(`Creating table: ${tableName}`);
+      this.createTable(tableName);
+    }
+
+    // Add missing columns
+    for (const { table, missingColumns } of results.incomplete) {
+      for (const column of missingColumns) {
+        console.log(`Adding column ${column} to ${table}`);
+        this.addColumn(table, column);
+      }
+    }
+
+    console.log('\n✅ Schema fixes applied\n');
+
+    return this.validateSchema();
+  }
+
+  createTable(tableName) {
+    const schemas = {
+      'memories': `
+        CREATE TABLE IF NOT EXISTS memories (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          key TEXT NOT NULL,
+          value TEXT NOT NULL,
+          namespace TEXT DEFAULT 'default',
+          created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+          updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+          expires_at DATETIME,
+          metadata TEXT
+        )
+      `,
+      'memory_embeddings': `
+        CREATE TABLE IF NOT EXISTS memory_embeddings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          memory_id INTEGER NOT NULL,
+          embedding TEXT NOT NULL,
+          created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+          FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+        )
+      `,
+      'sessions': `
+        CREATE TABLE IF NOT EXISTS sessions (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id TEXT UNIQUE NOT NULL,
+          agent_type TEXT,
+          started_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+          ended_at DATETIME,
+          metadata TEXT
+        )
+      `,
+      'session_metrics': `
+        CREATE TABLE IF NOT EXISTS session_metrics (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id TEXT NOT NULL,
+          metric_name TEXT NOT NULL,
+          metric_value REAL NOT NULL,
+          recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+          FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE
+        )
+      `,
+      'neural_patterns': `
+        CREATE TABLE IF NOT EXISTS neural_patterns (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          pattern_type TEXT NOT NULL,
+          pattern_data TEXT NOT NULL,
+          confidence REAL DEFAULT 0.5,
+          created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+      `,
+      'training_data': `
+        CREATE TABLE IF NOT EXISTS training_data (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          input TEXT NOT NULL,
+          output TEXT NOT NULL,
+          success INTEGER DEFAULT 1,
+          created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+      `
+    };
+
+    if (schemas[tableName]) {
+      this.db.exec(schemas[tableName]);
+      console.log(`✅ Created table: ${tableName}`);
+    }
+  }
+
+  addColumn(table, column) {
+    const columnTypes = {
+      'metadata': 'TEXT',
+      'expires_at': 'DATETIME',
+      'namespace': 'TEXT DEFAULT "default"'
+    };
+
+    const type = columnTypes[column] || 'TEXT';
+
+    try {
+      this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
+      console.log(`✅ Added column ${column} to ${table}`);
+    } catch (error) {
+      console.log(`⚠️  Could not add column ${column}: ${error.message}`);
+    }
+  }
+
+  createIndexes() {
+    console.log('\n🚀 Creating performance indexes...\n');
+
+    const indexes = [
+      // ReasoningBank indexes
+      'CREATE INDEX IF NOT EXISTS idx_patterns_confidence ON patterns(confidence DESC)',
+      'CREATE INDEX IF NOT EXISTS idx_patterns_domain ON patterns(domain)',
+      'CREATE INDEX IF NOT EXISTS idx_patterns_success ON patterns(success_rate DESC)',
+      'CREATE INDEX IF NOT EXISTS idx_embeddings_pattern ON pattern_embeddings(pattern_id)',
+      'CREATE INDEX IF NOT EXISTS idx_links_source ON pattern_links(source_id)',
+      'CREATE INDEX IF NOT EXISTS idx_links_target ON pattern_links(target_id)',
+      'CREATE INDEX IF NOT EXISTS idx_trajectories_memory ON task_trajectories(memory_id)',
+
+      // Claude-Flow indexes
+      'CREATE INDEX IF NOT EXISTS idx_memories_key ON memories(key)',
+      'CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories(namespace)',
+      'CREATE INDEX IF NOT EXISTS idx_memories_expires ON memories(expires_at)',
+      'CREATE INDEX IF NOT EXISTS idx_memory_emb_id ON memory_embeddings(memory_id)',
+      'CREATE INDEX IF NOT EXISTS idx_sessions_id ON sessions(session_id)',
+      'CREATE INDEX IF NOT EXISTS idx_metrics_session ON session_metrics(session_id)',
+      'CREATE INDEX IF NOT EXISTS idx_neural_type ON neural_patterns(pattern_type)'
+    ];
+
+    for (const index of indexes) {
+      this.db.exec(index);
+    }
+
+    console.log('✅ All indexes created\n');
+  }
+
+  optimizeDatabase() {
+    console.log('🔧 Optimizing database...\n');
+
+    // SQLite optimizations
+    this.db.exec('PRAGMA journal_mode=WAL');
+    this.db.exec('PRAGMA synchronous=NORMAL');
+    this.db.exec('PRAGMA cache_size=10000');
+    this.db.exec('PRAGMA temp_store=MEMORY');
+    this.db.exec('PRAGMA mmap_size=268435456');
+    this.db.exec('ANALYZE');
+    this.db.exec('VACUUM');
+
+    console.log('✅ Database optimized\n');
+  }
+
+  generateReport() {
+    const results = this.validateSchema();
+
+    const report = `
+# Schema Validation Report
+
+**Database**: ${this.dbPath}
+**Validation Time**: ${new Date().toISOString()}
+**Status**: ${results.valid ? '✅ VALID' : '❌ INVALID'}
+
+## Table Summary
+
+| Table | Status | Columns |
+|-------|--------|---------|
+${Object.entries(results.tables).map(([name, info]) =>
+  `| ${name} | ${info.exists ? (info.complete ? '✅' : '⚠️') : '❌'} | ${info.columns.length} |`
+).join('\n')}
+
+## Missing Tables
+${results.missing.length > 0 ? results.missing.map(t => `- ${t}`).join('\n') : '✅ None'}
+
+## Incomplete Tables
+${results.incomplete.length > 0 ? results.incomplete.map(i =>
+  `- **${i.table}**: Missing ${i.missingColumns.join(', ')}`
+).join('\n') : '✅ None'}
+
+## Database Statistics
+
+\`\`\`sql
+${this.getStatistics()}
+\`\`\`
+`;
+
+    return report;
+  }
+
+  getStatistics() {
+    const stats = [];
+
+    const tables = this.db.prepare(`
+      SELECT name FROM sqlite_master WHERE type='table'
+    `).all();
+
+    for (const { name } of tables) {
+      try {
+        const count = this.db.prepare(`SELECT COUNT(*) as count FROM ${name}`).get();
+        stats.push(`${name}: ${count.count} rows`);
+      } catch (error) {
+        stats.push(`${name}: error`);
+      }
+    }
+
+    return stats.join('\n');
+  }
+
+  close() {
+    this.db.close();
+  }
+}
+
+// CLI usage
+if (require.main === module) {
+  const dbPath = process.argv[2];
+  const action = process.argv[3] || 'validate';
+
+  if (!dbPath) {
+    console.error('Usage: node schema-validator.js <db-path> [validate|fix|report]');
+    process.exit(1);
+  }
+
+  const validator = new SchemaValidator(dbPath);
+
+  switch (action) {
+    case 'validate':
+      const results = validator.validateSchema();
+      validator.close();
+      process.exit(results.valid ? 0 : 1);
+      break;
+
+    case 'fix':
+      validator.createMissingTables();
+      validator.createIndexes();
+      validator.optimizeDatabase();
+      validator.close();
+      break;
+
+    case 'report':
+      console.log(validator.generateReport());
+      validator.close();
+      break;
+
+    default:
+      console.error('Unknown action. Use: validate, fix, or report');
+      process.exit(1);
+  }
+}
+
+module.exports = SchemaValidator;

package/docs/reasoningbank/models/_scripts/training-coordinator.cjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/**
+ * ReasoningBank Model Training Coordinator
+ * Orchestrates parallel training of 5 specialized models with memory coordination
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+class TrainingCoordinator {
+  constructor() {
+    this.swarmId = `training-swarm-${Date.now()}`;
+    this.models = {
+      'safla': { target: 2000, description: 'Self-Aware Feedback Loop patterns' },
+      'google-research': { target: 3000, description: 'Strategy-level memory patterns' },
+      'code-reasoning': { target: 2500, description: 'Programming best practices' },
+      'problem-solving': { target: 2000, description: 'General reasoning patterns' },
+      'domain-expert': { target: 1500, description: 'Multi-domain expertise' }
+    };
+  }
+
+  async initializeSwarm() {
+    console.log(`🚀 Initializing training swarm: ${this.swarmId}`);
+
+    // Store swarm coordination info in memory
+    execSync(`npx claude-flow@alpha memory store swarm/coordination "${JSON.stringify({
+      swarmId: this.swarmId,
+      agents: Object.keys(this.models),
+      startTime: new Date().toISOString(),
+      status: 'initializing'
+    })}" --namespace training --reasoningbank`, { stdio: 'inherit' });
+  }
+
+  async validateDatabase(modelPath) {
+    const dbPath = path.join(modelPath, 'memory.db');
+    if (!fs.existsSync(dbPath)) {
+      throw new Error(`Database not found: ${dbPath}`);
+    }
+
+    // Validate schema and count patterns
+    const result = execSync(
+      `sqlite3 ${dbPath} "SELECT COUNT(*) FROM patterns"`,
+      { encoding: 'utf-8' }
+    );
+
+    const count = parseInt(result.trim());
+    console.log(`✅ Validated ${modelPath}: ${count} patterns`);
+    return count;
+  }
+
+  async reportProgress(modelName, progress) {
+    // Store progress in shared memory
+    execSync(`npx claude-flow@alpha memory store swarm/progress/${modelName} "${JSON.stringify(progress)}" --namespace training --reasoningbank`, { stdio: 'inherit' });
+  }
+
+  async getSwarmStatus() {
+    // Query all agent progress
+    const result = execSync(
+      `npx claude-flow@alpha memory query "swarm/progress" --namespace training --reasoningbank`,
+      { encoding: 'utf-8' }
+    );
+    return result;
+  }
+
+  async finalizeSwarm() {
+    console.log(`✅ Training swarm ${this.swarmId} completed`);
+
+    // Store final status
+    execSync(`npx claude-flow@alpha memory store swarm/coordination "${JSON.stringify({
+      swarmId: this.swarmId,
+      endTime: new Date().toISOString(),
+      status: 'completed'
+    })}" --namespace training --reasoningbank`, { stdio: 'inherit' });
+  }
+}
+
+// Export for use by training agents
+module.exports = TrainingCoordinator;
+
+// CLI usage
+if (require.main === module) {
+  const coordinator = new TrainingCoordinator();
+  coordinator.initializeSwarm();
+}

package/docs/reasoningbank/models/_scripts/validation-suite.cjs

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+/**
+ * ReasoningBank Model Validation Suite
+ * Validates trained models before and after optimization
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+class ModelValidator {
+  constructor(modelPath, modelName) {
+    this.modelPath = modelPath;
+    this.modelName = modelName;
+    this.dbPath = path.join(modelPath, 'memory.db');
+  }
+
+  async runValidation() {
+    console.log(`\n🔍 Validating ${this.modelName} model...`);
+
+    const results = {
+      model: this.modelName,
+      timestamp: new Date().toISOString(),
+      checks: {}
+    };
+
+    // 1. Database exists
+    results.checks.databaseExists = fs.existsSync(this.dbPath);
+
+    // 2. Schema validation
+    results.checks.schema = this.validateSchema();
+
+    // 3. Pattern count
+    results.checks.patternCount = this.getPatternCount();
+
+    // 4. Embedding quality
+    results.checks.embeddings = this.validateEmbeddings();
+
+    // 5. Confidence scores
+    results.checks.confidence = this.validateConfidence();
+
+    // 6. Pattern links
+    results.checks.links = this.validateLinks();
+
+    // 7. Query performance
+    results.checks.performance = await this.benchmarkQueries();
+
+    // 8. Storage efficiency
+    results.checks.storage = this.analyzeStorage();
+
+    // Generate report
+    this.generateReport(results);
+
+    return results;
+  }
+
+  validateSchema() {
+    const tables = execSync(
+      `sqlite3 ${this.dbPath} "SELECT name FROM sqlite_master WHERE type='table'"`,
+      { encoding: 'utf-8' }
+    ).trim().split('\n');
+
+    const required = ['patterns', 'pattern_embeddings', 'task_trajectories', 'pattern_links'];
+    const hasAll = required.every(t => tables.includes(t));
+
+    return { valid: hasAll, tables: tables.length, required: 4 };
+  }
+
+  getPatternCount() {
+    const count = parseInt(execSync(
+      `sqlite3 ${this.dbPath} "SELECT COUNT(*) FROM patterns"`,
+      { encoding: 'utf-8' }
+    ).trim());
+
+    return { count, valid: count >= 1000 && count <= 10000 };
+  }
+
+  validateEmbeddings() {
+    const embCount = parseInt(execSync(
+      `sqlite3 ${this.dbPath} "SELECT COUNT(*) FROM pattern_embeddings"`,
+      { encoding: 'utf-8' }
+    ).trim());
+
+    const patCount = this.getPatternCount().count;
+
+    // Check embedding dimensions (should be 1024)
+    const sample = execSync(
+      `sqlite3 ${this.dbPath} "SELECT embedding FROM pattern_embeddings LIMIT 1"`,
+      { encoding: 'utf-8' }
+    ).trim();
+
+    const dims = sample ? JSON.parse(sample).length : 0;
+
+    return {
+      count: embCount,
+      dimensions: dims,
+      coverage: (embCount / patCount * 100).toFixed(2) + '%',
+      valid: embCount === patCount && dims === 1024
+    };
+  }
+
+  validateConfidence() {
+    const stats = execSync(
+      `sqlite3 ${this.dbPath} "SELECT AVG(confidence), MIN(confidence), MAX(confidence), COUNT(*) FROM patterns WHERE confidence > 0"`,
+      { encoding: 'utf-8' }
+    ).trim().split('|');
+
+    return {
+      average: parseFloat(stats[0]).toFixed(3),
+      min: parseFloat(stats[1]).toFixed(3),
+      max: parseFloat(stats[2]).toFixed(3),
+      withConfidence: parseInt(stats[3]),
+      valid: parseFloat(stats[0]) >= 0.5 && parseFloat(stats[0]) <= 1.0
+    };
+  }
+
+  validateLinks() {
+    const linkCount = parseInt(execSync(
+      `sqlite3 ${this.dbPath} "SELECT COUNT(*) FROM pattern_links"`,
+      { encoding: 'utf-8' }
+    ).trim());
+
+    const typeDistribution = execSync(
+      `sqlite3 ${this.dbPath} "SELECT link_type, COUNT(*) FROM pattern_links GROUP BY link_type"`,
+      { encoding: 'utf-8' }
+    ).trim();
+
+    return {
+      total: linkCount,
+      types: typeDistribution || 'none',
+      valid: linkCount > 0
+    };
+  }
+
+  async benchmarkQueries() {
+    const start = Date.now();
+
+    // Run 100 semantic queries
+    for (let i = 0; i < 100; i++) {
+      execSync(
+        `npx claude-flow@alpha memory query "test query ${i}" --namespace ${this.modelName} --reasoningbank`,
+        { stdio: 'pipe' }
+      );
+    }
+
+    const duration = Date.now() - start;
+    const avgLatency = duration / 100;
+
+    return {
+      queries: 100,
+      totalTime: duration + 'ms',
+      avgLatency: avgLatency.toFixed(2) + 'ms',
+      valid: avgLatency < 10 // Should be under 10ms
+    };
+  }
+
+  analyzeStorage() {
+    const stats = fs.statSync(this.dbPath);
+    const sizeKB = (stats.size / 1024).toFixed(2);
+    const patternCount = this.getPatternCount().count;
+    const perPattern = (stats.size / patternCount / 1024).toFixed(2);
+
+    return {
+      totalSize: sizeKB + ' KB',
+      patterns: patternCount,
+      perPattern: perPattern + ' KB',
+      valid: parseFloat(perPattern) < 10 // Should be under 10KB per pattern
+    };
+  }
+
+  generateReport(results) {
+    const report = `
+# Validation Report: ${results.model}
+**Generated**: ${results.timestamp}
+
+## ✅ Database Checks
+- Database exists: ${results.checks.databaseExists ? '✅' : '❌'}
+- Schema valid: ${results.checks.schema.valid ? '✅' : '❌'} (${results.checks.schema.tables}/4 tables)
+- Pattern count: ${results.checks.patternCount.valid ? '✅' : '❌'} (${results.checks.patternCount.count} patterns)
+
+## 📊 Data Quality
+- Embeddings: ${results.checks.embeddings.valid ? '✅' : '❌'}
+  - Coverage: ${results.checks.embeddings.coverage}
+  - Dimensions: ${results.checks.embeddings.dimensions}
+- Confidence scores: ${results.checks.confidence.valid ? '✅' : '❌'}
+  - Average: ${results.checks.confidence.average}
+  - Range: ${results.checks.confidence.min} - ${results.checks.confidence.max}
+- Pattern links: ${results.checks.links.valid ? '✅' : '❌'} (${results.checks.links.total} links)
+
+## ⚡ Performance
+- Query latency: ${results.checks.performance.valid ? '✅' : '❌'}
+  - Average: ${results.checks.performance.avgLatency}
+  - Queries tested: ${results.checks.performance.queries}
+
+## 💾 Storage Efficiency
+- Total size: ${results.checks.storage.totalSize}
+- Per pattern: ${results.checks.storage.perPattern}
+- Efficiency: ${results.checks.storage.valid ? '✅' : '❌'}
+
+---
+**Overall Status**: ${this.isValid(results) ? '✅ PASSED' : '❌ FAILED'}
+`;
+
+    fs.writeFileSync(
+      path.join(this.modelPath, 'validation-report.md'),
+      report
+    );
+
+    console.log(report);
+  }
+
+  isValid(results) {
+    return results.checks.databaseExists &&
+           results.checks.schema.valid &&
+           results.checks.patternCount.valid &&
+           results.checks.embeddings.valid &&
+           results.checks.confidence.valid &&
+           results.checks.performance.valid &&
+           results.checks.storage.valid;
+  }
+}
+
+module.exports = ModelValidator;
+
+// CLI usage
+if (require.main === module) {
+  const modelPath = process.argv[2];
+  const modelName = process.argv[3] || path.basename(modelPath);
+
+  if (!modelPath) {
+    console.error('Usage: node validation-suite.js <model-path> [model-name]');
+    process.exit(1);
+  }
+
+  const validator = new ModelValidator(modelPath, modelName);
+  validator.runValidation().then(results => {
+    process.exit(validator.isValid(results) ? 0 : 1);
+  });
+}