@equilateral_ai/mindmeld 3.0.0

package/src/core/StandardsIngestion.js

@@ -0,0 +1,575 @@
+/**
+ * StandardsIngestion.js - Phase 1 of Rapport Standards Integration
+ *
+ * Parses and ingests .equilateral-standards/ markdown files into Rapport database
+ * Creates standards_patterns from enforcement rules, anti-patterns, examples, and cost impacts
+ *
+ * Reference: /Users/jamesford/Source/rapport/docs/RAPPORT_STANDARDS_INTEGRATION_DESIGN.md
+ */
+
+const fs = require('fs').promises;
+const path = require('path');
+const { executeQuery } = require('../handlers/helpers/dbOperations');
+
+class StandardsIngestion {
+  constructor(options = {}) {
+    this.standardsPath = options.standardsPath || path.join(process.cwd(), '.equilateral-standards');
+    this.verbose = options.verbose || false;
+  }
+
+  /**
+   * Main ingestion entry point
+   * Discovers and parses all .equilateral-standards/ markdown files
+   */
+  async ingestEquilateralStandards() {
+    console.log(`[StandardsIngestion] Starting ingestion from ${this.standardsPath}`);
+
+    const startTime = Date.now();
+    const patterns = [];
+
+    // Define standard categories
+    const categories = [
+      'serverless-saas-aws',
+      'multi-agent-orchestration',
+      'compliance-security',
+      'database',
+      'frontend-development',
+      'cost-optimization',
+      'backend',
+      'deployment',
+      'real-time-systems',
+      'testing',
+      'well-architected'
+    ];
+
+    // Discover and parse all standards files
+    for (const category of categories) {
+      const categoryPath = path.join(this.standardsPath, category);
+
+      try {
+        const files = await this.discoverStandardFiles(categoryPath);
+
+        for (const file of files) {
+          try {
+            const standard = await this.parseStandard(file, category);
+
+            if (standard && standard.patterns.length > 0) {
+              patterns.push(...standard.patterns);
+
+              if (this.verbose) {
+                console.log(`  ✓ Parsed ${path.basename(file)}: ${standard.patterns.length} patterns`);
+              }
+            }
+          } catch (parseError) {
+            console.warn(`  ⚠ Failed to parse ${path.basename(file)}:`, parseError.message);
+          }
+        }
+      } catch (categoryError) {
+        if (categoryError.code !== 'ENOENT') {
+          console.warn(`  ⚠ Failed to process category ${category}:`, categoryError.message);
+        }
+      }
+    }
+
+    // Store in database
+    const ingestedCount = await this.seedPatternDatabase(patterns);
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+    return {
+      patternsIngested: ingestedCount,
+      totalPatternsFound: patterns.length,
+      categories: categories.length,
+      timestamp: new Date().toISOString(),
+      duration: `${duration}s`
+    };
+  }
+
+  /**
+   * Discover all markdown files in a category directory
+   */
+  async discoverStandardFiles(categoryPath) {
+    try {
+      const entries = await fs.readdir(categoryPath, { withFileTypes: true });
+      const files = [];
+
+      for (const entry of entries) {
+        const fullPath = path.join(categoryPath, entry.name);
+
+        if (entry.isDirectory()) {
+          // Recursively search subdirectories
+          const subFiles = await this.discoverStandardFiles(fullPath);
+          files.push(...subFiles);
+        } else if (entry.isFile() && entry.name.endsWith('.md')) {
+          files.push(fullPath);
+        }
+      }
+
+      return files;
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        // Category doesn't exist - not an error
+        return [];
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Parse a markdown standards file into Rapport pattern format
+   */
+  async parseStandard(filepath, category) {
+    const content = await fs.readFile(filepath, 'utf-8');
+    const filename = path.basename(filepath, '.md');
+
+    // Extract standard metadata from markdown
+    const title = this.extractTitle(content);
+    const enforcementRules = this.extractEnforcementRules(content);
+    const antiPatterns = this.extractAntiPatterns(content);
+    const examples = this.extractExamples(content);
+    const costImpact = this.extractCostImpact(content);
+    const applicableFiles = this.extractApplicableFiles(content, category);
+
+    const patterns = [];
+
+    // Create pattern for each enforcement rule
+    for (const rule of enforcementRules) {
+      const patternId = this.generatePatternId(category, filename, rule.element);
+
+      patterns.push({
+        pattern_id: patternId,
+        element: rule.element,
+        rule: rule.rule,
+        correlation: 1.0, // Standards are mandatory
+        source: 'equilateral-standards',
+        maturity: rule.severity === 'MANDATORY' || rule.severity === 'CRITICAL' ? 'enforced' : 'validated',
+        scope: 'organization',
+        category: category,
+        applicable_files: applicableFiles,
+        anti_patterns: antiPatterns,
+        examples: examples,
+        cost_impact: costImpact,
+        source_file: filepath,
+        title: title
+      });
+    }
+
+    return { patterns };
+  }
+
+  /**
+   * Extract title from markdown (first H1)
+   */
+  extractTitle(content) {
+    const match = content.match(/^#\s+(.+)$/m);
+    return match ? match[1].trim() : 'Untitled Standard';
+  }
+
+  /**
+   * Extract enforcement rules from markdown
+   * Looks for sections like "MANDATORY PATTERNS", "Core Principles", etc.
+   */
+  extractEnforcementRules(content) {
+    const rules = [];
+
+    // Pattern 1: MANDATORY/CRITICAL sections
+    const mandatorySection = content.match(/##?\s*⚠?️?\s*(MANDATORY|CRITICAL)[^#]*/gi);
+    if (mandatorySection) {
+      for (const section of mandatorySection) {
+        const sectionRules = this.extractRulesFromSection(section, 'MANDATORY');
+        rules.push(...sectionRules);
+      }
+    }
+
+    // Pattern 2: "ALWAYS" and "NEVER" statements
+    const alwaysNever = content.match(/(?:ALWAYS|NEVER)\s+([^.\n]+)/g);
+    if (alwaysNever) {
+      for (const statement of alwaysNever) {
+        rules.push({
+          element: this.extractElement(statement),
+          rule: statement.trim(),
+          severity: 'MANDATORY'
+        });
+      }
+    }
+
+    // Pattern 3: Core Principles
+    const principleMatch = content.match(/##?\s*Core Principle[s]?[:\s]+([^#]+)/i);
+    if (principleMatch) {
+      rules.push({
+        element: 'Core Principle',
+        rule: principleMatch[1].trim(),
+        severity: 'ENFORCED'
+      });
+    }
+
+    return rules.length > 0 ? rules : [{
+      element: 'Standard Pattern',
+      rule: this.extractFirstParagraph(content),
+      severity: 'VALIDATED'
+    }];
+  }
+
+  /**
+   * Extract rules from a section of content
+   */
+  extractRulesFromSection(section, severity) {
+    const rules = [];
+
+    // Look for numbered or bulleted lists
+    const listItems = section.match(/^[\s]*[-*\d]+\.?\s+(.+)$/gm);
+    if (listItems) {
+      for (const item of listItems) {
+        const cleaned = item.replace(/^[\s]*[-*\d]+\.?\s+/, '').trim();
+        if (cleaned.length > 10) { // Filter out very short items
+          rules.push({
+            element: this.extractElement(cleaned),
+            rule: cleaned,
+            severity: severity
+          });
+        }
+      }
+    }
+
+    // Look for bold statements
+    const boldStatements = section.match(/\*\*(.+?)\*\*/g);
+    if (boldStatements && rules.length === 0) {
+      for (const statement of boldStatements) {
+        const cleaned = statement.replace(/\*\*/g, '').trim();
+        if (cleaned.length > 10 && !cleaned.match(/^(MANDATORY|CRITICAL|NOTE|IMPORTANT)/i)) {
+          rules.push({
+            element: this.extractElement(cleaned),
+            rule: cleaned,
+            severity: severity
+          });
+        }
+      }
+    }
+
+    return rules;
+  }
+
+  /**
+   * Extract anti-patterns from markdown
+   * Looks for ❌ WRONG, NEVER DO THIS, Anti-Pattern sections
+   */
+  extractAntiPatterns(content) {
+    const antiPatterns = [];
+
+    // Pattern 1: ❌ markers
+    const wrongPatterns = content.match(/❌[^```]*?```[^`]+```/gs);
+    if (wrongPatterns) {
+      for (const pattern of wrongPatterns) {
+        const description = pattern.match(/❌\s*([^\n`]+)/);
+        const code = pattern.match(/```(\w+)?\n([\s\S]+?)```/);
+
+        if (description && code) {
+          antiPatterns.push({
+            description: description[1].trim(),
+            code: code[2].trim(),
+            language: code[1] || 'javascript'
+          });
+        }
+      }
+    }
+
+    // Pattern 2: Anti-Pattern sections
+    const antiPatternSection = content.match(/##?\s*Anti-Pattern[s]?[^#]*/gi);
+    if (antiPatternSection) {
+      for (const section of antiPatternSection) {
+        const sectionPatterns = this.extractCodeBlocksFromSection(section);
+        antiPatterns.push(...sectionPatterns.map(p => ({
+          description: p.description || 'Anti-pattern to avoid',
+          code: p.code,
+          language: p.language
+        })));
+      }
+    }
+
+    return antiPatterns;
+  }
+
+  /**
+   * Extract examples from markdown
+   * Looks for ✅ CORRECT, Example sections
+   */
+  extractExamples(content) {
+    const examples = [];
+
+    // Pattern 1: ✅ markers
+    const correctPatterns = content.match(/✅[^```]*?```[^`]+```/gs);
+    if (correctPatterns) {
+      for (const pattern of correctPatterns) {
+        const description = pattern.match(/✅\s*([^\n`]+)/);
+        const code = pattern.match(/```(\w+)?\n([\s\S]+?)```/);
+
+        if (description && code) {
+          examples.push({
+            description: description[1].trim(),
+            code: code[2].trim(),
+            language: code[1] || 'javascript'
+          });
+        }
+      }
+    }
+
+    // Pattern 2: Example sections
+    const exampleSection = content.match(/##?\s*Example[s]?[^#]*/gi);
+    if (exampleSection) {
+      for (const section of exampleSection) {
+        const sectionExamples = this.extractCodeBlocksFromSection(section);
+        examples.push(...sectionExamples.map(p => ({
+          description: p.description || 'Example implementation',
+          code: p.code,
+          language: p.language
+        })));
+      }
+    }
+
+    return examples;
+  }
+
+  /**
+   * Extract cost impact information
+   */
+  extractCostImpact(content) {
+    const costImpact = {};
+
+    // Look for cost comparison tables
+    const costTable = content.match(/\|[^|]*Cost[^|]*\|[\s\S]*?\n\n/i);
+    if (costTable) {
+      costImpact.hasTable = true;
+      costImpact.details = costTable[0].trim();
+    }
+
+    // Look for specific cost mentions
+    const costMentions = content.match(/\$\d+[^.\n]*/g);
+    if (costMentions) {
+      costImpact.mentions = costMentions.map(m => m.trim());
+    }
+
+    // Look for "Cost Impact" sections
+    const costSection = content.match(/##?\s*Cost Impact[^#]*/i);
+    if (costSection) {
+      costImpact.section = costSection[0].trim();
+    }
+
+    return Object.keys(costImpact).length > 0 ? costImpact : null;
+  }
+
+  /**
+   * Determine applicable file patterns based on category and content
+   */
+  extractApplicableFiles(content, category) {
+    const patterns = [];
+
+    // Category-based patterns
+    const categoryPatterns = {
+      'serverless-saas-aws': ['**/*.js', '**/template.yaml', '**/handlers/**/*.js'],
+      'frontend-development': ['**/*.tsx', '**/*.jsx', '**/*.ts', 'src/frontend/**/*'],
+      'database': ['**/*.sql', '**/dbOperations.js', '**/migrations/**/*'],
+      'backend': ['src/backend/**/*.js', 'src/handlers/**/*.js'],
+      'multi-agent-orchestration': ['src/agents/**/*.js', 'src/core/**/*.js']
+    };
+
+    if (categoryPatterns[category]) {
+      patterns.push(...categoryPatterns[category]);
+    }
+
+    // Content-based patterns
+    if (content.includes('Lambda') || content.includes('handler')) {
+      patterns.push('src/handlers/**/*.js');
+    }
+    if (content.includes('React') || content.includes('component')) {
+      patterns.push('src/frontend/**/*.tsx', 'src/frontend/**/*.jsx');
+    }
+    if (content.includes('PostgreSQL') || content.includes('database')) {
+      patterns.push('**/*dbOperations.js', '**/*.sql');
+    }
+
+    return [...new Set(patterns)]; // Remove duplicates
+  }
+
+  /**
+   * Extract code blocks from a section
+   */
+  extractCodeBlocksFromSection(section) {
+    const blocks = [];
+    const codeBlocks = section.match(/```(\w+)?\n([\s\S]+?)```/g);
+
+    if (codeBlocks) {
+      for (const block of codeBlocks) {
+        const match = block.match(/```(\w+)?\n([\s\S]+?)```/);
+        if (match) {
+          // Look for description before code block
+          const beforeBlock = section.substring(0, section.indexOf(block));
+          const descMatch = beforeBlock.match(/([^\n]+)\n*$/);
+
+          blocks.push({
+            description: descMatch ? descMatch[1].trim() : null,
+            code: match[2].trim(),
+            language: match[1] || 'javascript'
+          });
+        }
+      }
+    }
+
+    return blocks;
+  }
+
+  /**
+   * Extract element name from a rule statement
+   */
+  extractElement(statement) {
+    // Remove modifiers
+    let cleaned = statement.replace(/^(ALWAYS|NEVER|DO NOT|DON'T)\s+/i, '');
+
+    // Extract first noun phrase (simple heuristic)
+    const match = cleaned.match(/^([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)/);
+    if (match) {
+      return match[1];
+    }
+
+    // Extract from "Use X" or "Implement X"
+    const actionMatch = cleaned.match(/^(?:Use|Implement|Apply|Follow|Add|Create)\s+([^,.\n]+)/i);
+    if (actionMatch) {
+      return actionMatch[1].trim();
+    }
+
+    // Fallback: first few words
+    const words = cleaned.split(/\s+/).slice(0, 3);
+    return words.join(' ');
+  }
+
+  /**
+   * Extract first paragraph from content
+   */
+  extractFirstParagraph(content) {
+    const paragraphs = content.split(/\n\n+/);
+    for (const para of paragraphs) {
+      const cleaned = para.replace(/^#+\s+/, '').trim();
+      if (cleaned.length > 20 && !cleaned.startsWith('**')) {
+        return cleaned;
+      }
+    }
+    return 'Standard pattern';
+  }
+
+  /**
+   * Generate unique pattern ID
+   */
+  generatePatternId(category, filename, element) {
+    const categorySlug = category.replace(/[^a-z0-9]/gi, '_');
+    const fileSlug = filename.replace(/[^a-z0-9]/gi, '_');
+    const elementSlug = element.replace(/[^a-z0-9]/gi, '_').toLowerCase().substring(0, 30);
+
+    return `${categorySlug}_${fileSlug}_${elementSlug}`.toLowerCase();
+  }
+
+  /**
+   * Store patterns in database
+   */
+  async seedPatternDatabase(patterns) {
+    if (patterns.length === 0) {
+      console.log('[StandardsIngestion] No patterns to ingest');
+      return 0;
+    }
+
+    console.log(`[StandardsIngestion] Storing ${patterns.length} patterns in database...`);
+
+    let successCount = 0;
+
+    for (const pattern of patterns) {
+      try {
+        await executeQuery(`
+          INSERT INTO rapport.standards_patterns (
+            pattern_id,
+            element,
+            rule,
+            correlation,
+            source,
+            maturity,
+            scope,
+            category,
+            applicable_files,
+            anti_patterns,
+            examples,
+            cost_impact,
+            created_at,
+            last_updated
+          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW(), NOW())
+          ON CONFLICT (pattern_id)
+          DO UPDATE SET
+            element = EXCLUDED.element,
+            rule = EXCLUDED.rule,
+            correlation = EXCLUDED.correlation,
+            maturity = EXCLUDED.maturity,
+            scope = EXCLUDED.scope,
+            category = EXCLUDED.category,
+            applicable_files = EXCLUDED.applicable_files,
+            anti_patterns = EXCLUDED.anti_patterns,
+            examples = EXCLUDED.examples,
+            cost_impact = EXCLUDED.cost_impact,
+            last_updated = NOW()
+        `, [
+          pattern.pattern_id,
+          pattern.element,
+          pattern.rule,
+          pattern.correlation,
+          pattern.source,
+          pattern.maturity,
+          pattern.scope,
+          pattern.category,
+          pattern.applicable_files,
+          JSON.stringify(pattern.anti_patterns),
+          JSON.stringify(pattern.examples),
+          pattern.cost_impact ? JSON.stringify(pattern.cost_impact) : null
+        ]);
+
+        successCount++;
+
+        if (this.verbose && successCount % 10 === 0) {
+          console.log(`  Stored ${successCount}/${patterns.length} patterns...`);
+        }
+      } catch (error) {
+        console.error(`  ✗ Failed to store pattern ${pattern.pattern_id}:`, error.message);
+      }
+    }
+
+    console.log(`[StandardsIngestion] ✓ Successfully stored ${successCount} patterns`);
+    return successCount;
+  }
+
+  /**
+   * Check if standards need re-ingestion (for performance)
+   */
+  async needsIngestion() {
+    try {
+      const result = await executeQuery(`
+        SELECT COUNT(*) as count, MAX(last_updated) as last_updated
+        FROM rapport.standards_patterns
+        WHERE source = 'equilateral-standards'
+      `);
+
+      const row = result.rows[0];
+
+      // Re-ingest if no patterns exist or haven't been updated in 7 days
+      if (row.count === '0') {
+        return { needed: true, reason: 'No standards ingested yet' };
+      }
+
+      const lastUpdated = new Date(row.last_updated);
+      const daysSince = (Date.now() - lastUpdated.getTime()) / (1000 * 60 * 60 * 24);
+
+      if (daysSince > 7) {
+        return { needed: true, reason: `Last updated ${daysSince.toFixed(0)} days ago` };
+      }
+
+      return { needed: false, count: row.count };
+    } catch (error) {
+      return { needed: true, reason: 'Database check failed' };
+    }
+  }
+}
+
+module.exports = { StandardsIngestion };