@equilateral_ai/mindmeld 3.1.2 → 3.3.0

package/src/core/StandardsIngestion.js

@@ -1,14 +1,23 @@
 /**
  * StandardsIngestion.js - Phase 1 of Rapport Standards Integration
  *
- * Parses and ingests .equilateral-standards/ markdown files into Rapport database
- * Creates standards_patterns from enforcement rules, anti-patterns, examples, and cost impacts
+ * Parses and ingests .equilateral-standards/ YAML files into Rapport database
+ * Creates standards_patterns from rules, anti-patterns, examples, and cost impacts
+ *
+ * YAML Schema (equilateral-standards-yaml):
+ *   id, category, priority (10/20/30), updated
+ *   rules: [{ action: ALWAYS|NEVER|USE|PREFER|AVOID, rule: string, applies_to: [globs] }]
+ *   anti_patterns: [strings]
+ *   examples: { name: code }
+ *   cost_impact: { pattern: impact }
+ *   tags: [strings]
  *
  * Reference: /Users/jamesford/Source/rapport/docs/RAPPORT_STANDARDS_INTEGRATION_DESIGN.md
  */
 
 const fs = require('fs').promises;
 const path = require('path');
+const yaml = require('js-yaml');
 const { executeQuery } = require('../handlers/helpers/dbOperations');
 
 class StandardsIngestion {
@@ -19,7 +28,7 @@ class StandardsIngestion {
 
   /**
    * Main ingestion entry point
-   * Discovers and parses all .equilateral-standards/ markdown files
+   * Discovers and parses all .equilateral-standards/ YAML files (with markdown fallback)
    */
   async ingestEquilateralStandards() {
     console.log(`[StandardsIngestion] Starting ingestion from ${this.standardsPath}`);
@@ -86,12 +95,61 @@ class StandardsIngestion {
   }
 
   /**
-   * Discover all markdown files in a category directory
+   * Parse all standards from YAML files without storing in database.
+   * Used as a fallback when the database is unavailable (e.g., local dev).
+   * @returns {Promise<Array>} Parsed patterns array
+   */
+  async parseAllStandards() {
+    const patterns = [];
+
+    const categories = [
+      'serverless-saas-aws',
+      'multi-agent-orchestration',
+      'compliance-security',
+      'database',
+      'frontend-development',
+      'cost-optimization',
+      'backend',
+      'deployment',
+      'real-time-systems',
+      'testing',
+      'well-architected'
+    ];
+
+    for (const category of categories) {
+      const categoryPath = path.join(this.standardsPath, category);
+
+      try {
+        const files = await this.discoverStandardFiles(categoryPath);
+
+        for (const file of files) {
+          try {
+            const standard = await this.parseStandard(file, category);
+            if (standard && standard.patterns.length > 0) {
+              patterns.push(...standard.patterns);
+            }
+          } catch (parseError) {
+            // Skip unparseable files
+          }
+        }
+      } catch (categoryError) {
+        if (categoryError.code !== 'ENOENT') {
+          // Skip missing categories silently
+        }
+      }
+    }
+
+    return patterns;
+  }
+
+  /**
+   * Discover all standard files in a category directory
+   * Prefers YAML files over markdown - if both exist for the same base name, only YAML is returned
    */
   async discoverStandardFiles(categoryPath) {
     try {
       const entries = await fs.readdir(categoryPath, { withFileTypes: true });
-      const files = [];
+      const fileMap = new Map(); // baseName -> { yaml: path, md: path }
 
       for (const entry of entries) {
         const fullPath = path.join(categoryPath, entry.name);
@@ -99,9 +157,47 @@ class StandardsIngestion {
         if (entry.isDirectory()) {
           // Recursively search subdirectories
           const subFiles = await this.discoverStandardFiles(fullPath);
-          files.push(...subFiles);
-        } else if (entry.isFile() && entry.name.endsWith('.md')) {
-          files.push(fullPath);
+          // Add subdirectory files directly (they've already been deduplicated)
+          for (const subFile of subFiles) {
+            const subBaseName = path.basename(subFile).replace(/\.(yaml|yml|md)$/, '');
+            const subKey = `${path.dirname(subFile)}/${subBaseName}`;
+            if (!fileMap.has(subKey)) {
+              fileMap.set(subKey, {});
+            }
+            if (subFile.endsWith('.yaml') || subFile.endsWith('.yml')) {
+              fileMap.get(subKey).yaml = subFile;
+            } else if (subFile.endsWith('.md')) {
+              fileMap.get(subKey).md = subFile;
+            }
+          }
+        } else if (entry.isFile()) {
+          const isYaml = entry.name.endsWith('.yaml') || entry.name.endsWith('.yml');
+          const isMd = entry.name.endsWith('.md');
+
+          if (isYaml || isMd) {
+            const baseName = entry.name.replace(/\.(yaml|yml|md)$/, '');
+            const key = `${categoryPath}/${baseName}`;
+
+            if (!fileMap.has(key)) {
+              fileMap.set(key, {});
+            }
+
+            if (isYaml) {
+              fileMap.get(key).yaml = fullPath;
+            } else {
+              fileMap.get(key).md = fullPath;
+            }
+          }
+        }
+      }
+
+      // Build final list, preferring YAML over markdown
+      const files = [];
+      for (const paths of fileMap.values()) {
+        if (paths.yaml) {
+          files.push(paths.yaml);
+        } else if (paths.md) {
+          files.push(paths.md);
         }
       }
 
@@ -116,9 +212,164 @@ class StandardsIngestion {
   }
 
   /**
-   * Parse a markdown standards file into Rapport pattern format
+   * Parse a standards file into Rapport pattern format
+   * Detects file type and routes to appropriate parser
    */
   async parseStandard(filepath, category) {
+    const isYaml = filepath.endsWith('.yaml') || filepath.endsWith('.yml');
+
+    if (isYaml) {
+      return this.parseYamlStandard(filepath, category);
+    }
+
+    return this.parseMarkdownStandard(filepath, category);
+  }
+
+  /**
+   * Parse a YAML standards file into Rapport pattern format
+   * YAML format:
+   *   id: string
+   *   category: string
+   *   priority: 10 | 20 | 30
+   *   rules:
+   *     - action: ALWAYS | NEVER | USE | PREFER | AVOID
+   *       rule: string
+   *       applies_to: [glob patterns]
+   *   anti_patterns:
+   *     - string
+   */
+  async parseYamlStandard(filepath, category) {
+    const content = await fs.readFile(filepath, 'utf-8');
+    const ext = filepath.endsWith('.yml') ? '.yml' : '.yaml';
+    const filename = path.basename(filepath, ext);
+
+    let doc;
+    try {
+      doc = yaml.load(content);
+    } catch (parseError) {
+      throw new Error(`YAML parse error: ${parseError.message}`);
+    }
+
+    if (!doc || typeof doc !== 'object') {
+      throw new Error('Invalid YAML structure: expected an object');
+    }
+
+    const patterns = [];
+    const standardId = doc.id || filename;
+    const standardCategory = doc.category || category;
+    const standardPriority = doc.priority || 30;
+
+    // Extract anti-patterns as simple strings
+    const antiPatterns = Array.isArray(doc.anti_patterns)
+      ? doc.anti_patterns.map(ap => ({
+          description: typeof ap === 'string' ? ap : ap.description || 'Anti-pattern to avoid',
+          code: typeof ap === 'object' ? ap.code : null,
+          language: typeof ap === 'object' ? ap.language : null
+        }))
+      : [];
+
+    // Extract examples - now a named object in YAML format
+    const examples = [];
+    if (doc.examples && typeof doc.examples === 'object') {
+      for (const [name, code] of Object.entries(doc.examples)) {
+        examples.push({
+          description: name.replace(/_/g, ' '),
+          code: typeof code === 'string' ? code.trim() : String(code),
+          language: this.inferLanguageFromCode(code)
+        });
+      }
+    }
+
+    // Extract cost_impact - now a structured object in YAML format
+    const costImpact = doc.cost_impact && typeof doc.cost_impact === 'object'
+      ? doc.cost_impact
+      : (typeof doc.cost_impact === 'string' ? { description: doc.cost_impact } : null);
+
+    // Extract tags for metadata
+    const tags = Array.isArray(doc.tags) ? doc.tags : [];
+
+    // Process each rule
+    const rules = Array.isArray(doc.rules) ? doc.rules : [];
+
+    for (let ruleIndex = 0; ruleIndex < rules.length; ruleIndex++) {
+      const rule = rules[ruleIndex];
+      if (!rule || typeof rule !== 'object') continue;
+
+      const action = rule.action || 'USE';
+      const ruleText = rule.rule || '';
+      const appliesTo = Array.isArray(rule.applies_to) ? rule.applies_to : [];
+
+      // Construct element name from action and first few words
+      const element = this.extractElement(`${action} ${ruleText}`);
+
+      // Construct full rule statement
+      const fullRule = `${action}: ${ruleText}`;
+
+      const patternId = this.generatePatternId(standardCategory, standardId, element, ruleIndex);
+
+      // Map action to maturity
+      const actionMaturityMap = {
+        'ALWAYS': 'enforced',
+        'NEVER': 'enforced',
+        'USE': 'validated',
+        'PREFER': 'validated',
+        'AVOID': 'validated'
+      };
+      const maturity = actionMaturityMap[action] || 'validated';
+
+      patterns.push({
+        pattern_id: patternId,
+        file_name: filename,
+        element: element,
+        rule: fullRule,
+        priority: standardPriority,
+        correlation: 1.0,
+        source: 'equilateral-standards',
+        maturity: maturity,
+        scope: 'organization',
+        category: standardCategory,
+        applicable_files: appliesTo.length > 0 ? appliesTo : this.extractApplicableFiles('', standardCategory),
+        anti_patterns: antiPatterns,
+        examples: examples,
+        cost_impact: costImpact,
+        tags: tags,
+        source_file: filepath,
+        title: standardId
+      });
+    }
+
+    // If no rules found but document exists, create a single pattern from the document
+    if (patterns.length === 0 && doc.id) {
+      const patternId = this.generatePatternId(standardCategory, standardId, 'standard_pattern');
+
+      patterns.push({
+        pattern_id: patternId,
+        file_name: filename,
+        element: 'Standard Pattern',
+        rule: `Standard: ${standardId}`,
+        priority: standardPriority,
+        correlation: 1.0,
+        source: 'equilateral-standards',
+        maturity: 'validated',
+        scope: 'organization',
+        category: standardCategory,
+        applicable_files: this.extractApplicableFiles('', standardCategory),
+        anti_patterns: antiPatterns,
+        examples: examples,
+        cost_impact: costImpact,
+        tags: tags,
+        source_file: filepath,
+        title: standardId
+      });
+    }
+
+    return { patterns };
+  }
+
+  /**
+   * Parse a markdown standards file into Rapport pattern format
+   */
+  async parseMarkdownStandard(filepath, category) {
     const content = await fs.readFile(filepath, 'utf-8');
     const filename = path.basename(filepath, '.md');
 
@@ -136,10 +387,21 @@ class StandardsIngestion {
     for (const rule of enforcementRules) {
       const patternId = this.generatePatternId(category, filename, rule.element);
 
+      // Map severity to priority (10=high, 20=medium, 30=low per DB constraint)
+      const priorityMap = {
+        'CRITICAL': 10,
+        'MANDATORY': 10,
+        'ENFORCED': 20,
+        'VALIDATED': 30
+      };
+      const priority = priorityMap[rule.severity] || 30;
+
       patterns.push({
         pattern_id: patternId,
+        file_name: filename, // Source markdown file name (without .md)
         element: rule.element,
         rule: rule.rule,
+        priority: priority,
         correlation: 1.0, // Standards are mandatory
         source: 'equilateral-standards',
         maturity: rule.severity === 'MANDATORY' || rule.severity === 'CRITICAL' ? 'enforced' : 'validated',
@@ -417,28 +679,63 @@ class StandardsIngestion {
     return blocks;
   }
 
+  /**
+   * Infer programming language from code content
+   */
+  inferLanguageFromCode(code) {
+    if (typeof code !== 'string') return 'text';
+
+    // YAML/SAM template patterns
+    if (code.includes('{{resolve:ssm:') || code.match(/^\s*(Environment|Resources|Properties):/m)) {
+      return 'yaml';
+    }
+    // JavaScript patterns
+    if (code.includes('async ') || code.includes('await ') || code.includes('const ') ||
+        code.includes('function ') || code.includes('=>') || code.includes('require(')) {
+      return 'javascript';
+    }
+    // SQL patterns
+    if (code.match(/\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b/i)) {
+      return 'sql';
+    }
+    // Python patterns
+    if (code.includes('def ') || code.includes('import ') || code.match(/^\s*class\s+\w+:/m)) {
+      return 'python';
+    }
+
+    return 'text';
+  }
+
   /**
    * Extract element name from a rule statement
+   * Goal: Create a descriptive, unique element name from the rule
    */
   extractElement(statement) {
-    // Remove modifiers
-    let cleaned = statement.replace(/^(ALWAYS|NEVER|DO NOT|DON'T)\s+/i, '');
-
-    // Extract first noun phrase (simple heuristic)
-    const match = cleaned.match(/^([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)/);
-    if (match) {
-      return match[1];
+    // Remove action prefix (ALWAYS, NEVER, USE, etc.)
+    let cleaned = statement.replace(/^(ALWAYS|NEVER|USE|PREFER|AVOID|DO NOT|DON'T)[:\s]+/i, '').trim();
+
+    // Extract key technical terms for better element names
+    // Pattern: "verb + object" -> extract the object/concept being acted upon
+    // Allow method calls like "client.end()" by not stopping at dots within identifiers
+    const verbObjectMatch = cleaned.match(/^(?:Use|Implement|Apply|Follow|Add|Create|Set|Cache|Validate|Configure|Enable|Disable|Fetch|Call)\s+(.+?)(?:\s+-\s+|,|$)/i);
+    if (verbObjectMatch) {
+      let object = verbObjectMatch[1].trim();
+      // Clean up trailing punctuation but keep method calls
+      object = object.replace(/\s*[-–].*$/, ''); // Remove explanatory text after dash
+      return object.substring(0, 50);
     }
 
-    // Extract from "Use X" or "Implement X"
-    const actionMatch = cleaned.match(/^(?:Use|Implement|Apply|Follow|Add|Create)\s+([^,.\n]+)/i);
-    if (actionMatch) {
-      return actionMatch[1].trim();
+    // For statements starting with a noun phrase, extract more context
+    const nounPhraseMatch = cleaned.match(/^([A-Za-z][a-z]+(?:[\s.][a-z()]+){0,5})/i);
+    if (nounPhraseMatch) {
+      let phrase = nounPhraseMatch[1].trim();
+      phrase = phrase.replace(/\s*[-–].*$/, ''); // Remove explanatory text
+      return phrase.substring(0, 50);
     }
 
-    // Fallback: first few words
-    const words = cleaned.split(/\s+/).slice(0, 3);
-    return words.join(' ');
+    // Fallback: first meaningful words
+    const words = cleaned.split(/\s+/).slice(0, 4);
+    return words.join(' ').substring(0, 50);
   }
 
   /**
@@ -457,13 +754,20 @@ class StandardsIngestion {
 
   /**
    * Generate unique pattern ID
+   * @param {string} category - Category name
+   * @param {string} filename - Source filename
+   * @param {string} element - Element description
+   * @param {number} index - Rule index for uniqueness
    */
-  generatePatternId(category, filename, element) {
+  generatePatternId(category, filename, element, index = 0) {
     const categorySlug = category.replace(/[^a-z0-9]/gi, '_');
     const fileSlug = filename.replace(/[^a-z0-9]/gi, '_');
-    const elementSlug = element.replace(/[^a-z0-9]/gi, '_').toLowerCase().substring(0, 30);
+    const elementSlug = element.replace(/[^a-z0-9]/gi, '_').toLowerCase().substring(0, 40);
+
+    // Include index for uniqueness when elements might collide
+    const suffix = index > 0 ? `_${index}` : '';
 
-    return `${categorySlug}_${fileSlug}_${elementSlug}`.toLowerCase();
+    return `${categorySlug}_${fileSlug}_${elementSlug}${suffix}`.toLowerCase();
   }
 
   /**
@@ -484,8 +788,10 @@ class StandardsIngestion {
         await executeQuery(`
           INSERT INTO rapport.standards_patterns (
             pattern_id,
+            file_name,
             element,
             rule,
+            priority,
             correlation,
             source,
             maturity,
@@ -495,13 +801,16 @@ class StandardsIngestion {
             anti_patterns,
             examples,
             cost_impact,
+            keywords,
             created_at,
             last_updated
-          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW(), NOW())
+          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), NOW())
           ON CONFLICT (pattern_id)
           DO UPDATE SET
+            file_name = EXCLUDED.file_name,
             element = EXCLUDED.element,
             rule = EXCLUDED.rule,
+            priority = EXCLUDED.priority,
             correlation = EXCLUDED.correlation,
             maturity = EXCLUDED.maturity,
             scope = EXCLUDED.scope,
@@ -510,11 +819,14 @@ class StandardsIngestion {
             anti_patterns = EXCLUDED.anti_patterns,
             examples = EXCLUDED.examples,
             cost_impact = EXCLUDED.cost_impact,
+            keywords = EXCLUDED.keywords,
             last_updated = NOW()
         `, [
           pattern.pattern_id,
+          pattern.file_name,
           pattern.element,
           pattern.rule,
+          pattern.priority,
           pattern.correlation,
           pattern.source,
           pattern.maturity,
@@ -523,7 +835,8 @@ class StandardsIngestion {
           pattern.applicable_files,
           JSON.stringify(pattern.anti_patterns),
           JSON.stringify(pattern.examples),
-          pattern.cost_impact ? JSON.stringify(pattern.cost_impact) : null
+          pattern.cost_impact ? JSON.stringify(pattern.cost_impact) : null,
+          pattern.tags || []
         ]);
 
         successCount++;