fraim-framework 2.0.34 → 2.0.36

package/registry/scripts/validate-test-coverage.ts

@@ -1,280 +1,280 @@
-#!/usr/bin/env node
-/**
- * Validation Plan Coverage Checker
- * 
- * Extracts validation plan from spec/design documents and verifies evidence coverage.
- * Used by Gate 9 in code-quality-check.sh
- * 
- * Usage: npx tsx <this-script> <issue-number>
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-interface ValidationScenario {
-  userScenario: string;
-  expectedOutcome: string;
-  validationMethod: string;
-}
-
-interface CoverageResult {
-  complete: boolean;
-  totalScenarios: number;
-  coveredScenarios: number;
-  missingScenarios: ValidationScenario[];
-  coverage: string; // "X/Y scenarios (Z%)"
-}
-
-/**
- * Extract issue number from branch name
- */
-function extractIssueNumberFromBranch(): number | null {
-  const branchName = process.env.GIT_BRANCH || '';
-  const match = branchName.match(/feature\/(\d+)/);
-  return match ? parseInt(match[1]) : null;
-}
-
-/**
- * Find spec or design file for an issue
- */
-function findSpecOrDesignFile(issueNumber: number): string | null {
-  const specDir = path.join(process.cwd(), 'docs', 'feature specs');
-  const rfcDir = path.join(process.cwd(), 'docs', 'rfcs');
-
-  // Try feature specs first
-  if (fs.existsSync(specDir)) {
-    const files = fs.readdirSync(specDir);
-    const specFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
-    if (specFile) {
-      return path.join(specDir, specFile);
-    }
-  }
-
-  // Then try RFCs
-  if (fs.existsSync(rfcDir)) {
-    const files = fs.readdirSync(rfcDir);
-    const rfcFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
-    if (rfcFile) {
-      return path.join(rfcDir, rfcFile);
-    }
-  }
-
-  return null;
-}
-
-/**
- * Extract validation plan table from markdown file
- */
-function extractValidationPlan(filePath: string): ValidationScenario[] {
-  const content = fs.readFileSync(filePath, 'utf-8');
-  const scenarios: ValidationScenario[] = [];
-
-  // Find "## Validation Plan" section (not "Validation Plan Coverage")
-  const validationPlanMatch = content.match(/##\s+Validation Plan\s*\n[\s\S]*?(?=\n##|$)/i);
-  if (!validationPlanMatch) {
-    return scenarios;
-  }
-
-  const validationSection = validationPlanMatch[0];
-
-  // Look for markdown table - find table header row with "User Scenario"
-  // Pattern: | User Scenario | Expected outcome | Validation method |
-  const tablePattern = /\|[\s\*]*User Scenario[\s\*]*\|[\s\*]*Expected outcome[\s\*]*\|[\s\*]*Validation method[\s\*]*\|[\s]*\n\|[\s:\-|]+\|[\s]*\n((?:\|.*?\|[\s]*\n)+)/i;
-  const tableMatch = validationSection.match(tablePattern);
-
-  if (!tableMatch) {
-    return scenarios;
-  }
-
-  const tableRows = tableMatch[1].trim().split('\n').filter(row => row.trim().startsWith('|'));
-
-  for (const row of tableRows) {
-    // Skip separator rows (all dashes/pipes)
-    if (row.match(/^\|[\s:\-|]+\|$/)) {
-      continue;
-    }
-
-    // Parse table row: | Scenario | Outcome | Method |
-    // Handle both formats: | **bold** | text | and | text | text |
-    const cells = row.split('|').map(c => c.trim()).filter(c => c && c !== '');
-
-    if (cells.length >= 2) {
-      // Remove markdown bold (**text**) from cells
-      const cleanCell = (cell: string): string => {
-        return cell
-          .replace(/^\*\*/, '')
-          .replace(/\*\*$/, '')
-          .replace(/^\*\*/g, '')
-          .replace(/\*\*/g, '')
-          .trim();
-      };
-
-      const userScenario = cleanCell(cells[0] || '');
-      const expectedOutcome = cleanCell(cells[1] || '');
-      const validationMethod = cleanCell(cells[2] || 'Manual validation');
-
-      // Skip header rows
-      const scenarioLower = userScenario.toLowerCase();
-      const outcomeLower = expectedOutcome.toLowerCase();
-      if (scenarioLower.includes('user scenario') ||
-        (scenarioLower.includes('scenario') && outcomeLower.includes('expected')) ||
-        scenarioLower === 'user scenario' ||
-        outcomeLower === 'expected outcome') {
-        continue;
-      }
-
-      // Skip empty rows or separator rows
-      if (!userScenario || userScenario === '-' || userScenario.length === 0) {
-        continue;
-      }
-
-      scenarios.push({
-        userScenario,
-        expectedOutcome,
-        validationMethod
-      });
-    }
-  }
-
-  return scenarios;
-}
-
-/**
- * Check if evidence exists for a scenario
- * Evidence can be in PR comments, test.log, or other sources
- */
-function checkEvidenceForScenario(scenario: ValidationScenario, issueNumber: number): boolean {
-  // Extract key terms from scenario for searching
-  const scenarioKeywords = scenario.userScenario
-    .toLowerCase()
-    .replace(/\*\*/g, '')
-    .split(/\s+/)
-    .filter(w => w.length > 3 && !['agent', 'the', 'with', 'without'].includes(w));
-
-  const outcomeKeywords = scenario.expectedOutcome
-    .toLowerCase()
-    .replace(/\*\*/g, '')
-    .split(/\s+/)
-    .filter(w => w.length > 3);
-
-  // Method 1: Check test.log for scenario keywords
-  const testLogPath = path.join(process.cwd(), 'test.log');
-  if (fs.existsSync(testLogPath)) {
-    const testLogContent = fs.readFileSync(testLogPath, 'utf-8').toLowerCase();
-    // Check if multiple keywords from scenario appear
-    const matchingKeywords = scenarioKeywords.filter(kw => testLogContent.includes(kw));
-    if (matchingKeywords.length >= 2) {
-      return true;
-    }
-  }
-
-  // Method 2: Check for evidence file (created by agent with scenario validation)
-  const evidenceFiles = ['test-evidence.md', 'validation-evidence.md', 'implementation-evidence.md'];
-  for (const evidenceFile of evidenceFiles) {
-    const evidencePath = path.join(process.cwd(), evidenceFile);
-    if (fs.existsSync(evidencePath)) {
-      const evidenceContent = fs.readFileSync(evidencePath, 'utf-8').toLowerCase();
-      const matchingKeywords = scenarioKeywords.filter(kw => evidenceContent.includes(kw));
-      if (matchingKeywords.length >= 2) {
-        return true;
-      }
-    }
-  }
-
-  // Method 3: Check server.log for scenario-related activity
-  const serverLogPath = path.join(process.cwd(), 'server.log');
-  if (fs.existsSync(serverLogPath)) {
-    const serverLogContent = fs.readFileSync(serverLogPath, 'utf-8').toLowerCase();
-    // Look for validation-related keywords
-    if (scenarioKeywords.some(kw => serverLogContent.includes(kw)) ||
-      outcomeKeywords.some(kw => serverLogContent.includes('gate') && serverLogContent.includes(kw))) {
-      return true;
-    }
-  }
-
-  // Note: PR comment checking is done in the shell script (Gate 9)
-  // This TypeScript tool focuses on file-based evidence
-
-  return false; // Conservative: require evidence
-}
-
-/**
- * Main function
- */
-function main(): void {
-  const issueNumberArg = process.argv[2];
-  const issueNumber = issueNumberArg ? parseInt(issueNumberArg) : extractIssueNumberFromBranch();
-
-  if (!issueNumber) {
-    console.error('❌ ERROR: Could not determine issue number');
-    console.error('   Usage: npx tsx validate-coverage.ts <issue-number>');
-    console.error('   Or set GIT_BRANCH environment variable with feature/N-* format');
-    process.exit(1);
-  }
-
-  const specFile = findSpecOrDesignFile(issueNumber);
-
-  if (!specFile) {
-    // No spec/design found - Gate 9 warns but doesn't block
-    console.log('ℹ️  No spec/design found for issue #' + issueNumber);
-    console.log('   Skipping validation plan coverage check');
-    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no spec/design)');
-    process.exit(0);
-  }
-
-  console.log(`Found spec/design: ${specFile}`);
-
-  const scenarios = extractValidationPlan(specFile);
-
-  if (scenarios.length === 0) {
-    console.log('ℹ️  No validation plan table found in spec/design');
-    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no validation plan)');
-    process.exit(0);
-  }
-
-  console.log(`Found ${scenarios.length} validation scenario(s)`);
-
-  // Check evidence for each scenario
-  const covered: ValidationScenario[] = [];
-  const missing: ValidationScenario[] = [];
-
-  for (const scenario of scenarios) {
-    if (checkEvidenceForScenario(scenario, issueNumber)) {
-      covered.push(scenario);
-    } else {
-      missing.push(scenario);
-    }
-  }
-
-  const coverage: CoverageResult = {
-    complete: missing.length === 0,
-    totalScenarios: scenarios.length,
-    coveredScenarios: covered.length,
-    missingScenarios: missing,
-    coverage: `${covered.length}/${scenarios.length} scenarios (${Math.round(covered.length / scenarios.length * 100)}%)`
-  };
-
-  // Output results
-  console.log('\n📊 Validation Plan Coverage:');
-  console.log(`   ${coverage.coverage}`);
-
-  if (coverage.complete) {
-    console.log('✅ PASSED: All validation scenarios have evidence');
-    console.log('   Gate 9: Validation Plan Coverage - PASSED');
-    process.exit(0);
-  } else {
-    console.log('❌ FAILED: Missing evidence for validation scenarios:');
-    for (const scenario of missing) {
-      console.log(`   - "${scenario.userScenario}"`);
-      console.log(`     Expected: ${scenario.expectedOutcome}`);
-      console.log(`     Method: ${scenario.validationMethod}`);
-    }
-    console.log('\n   Required: Provide evidence for all validation scenarios before marking PR ready');
-    console.log('   Evidence can be: PR comment, test output, manual verification log');
-    console.log('   Gate 9: Validation Plan Coverage - FAILED');
-    process.exit(1);
-  }
-}
-
-// Run main function
-main();
+#!/usr/bin/env node
+/**
+ * Validation Plan Coverage Checker
+ * 
+ * Extracts validation plan from spec/design documents and verifies evidence coverage.
+ * Used by Gate 9 in code-quality-check.sh
+ * 
+ * Usage: npx tsx <this-script> <issue-number>
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface ValidationScenario {
+  userScenario: string;
+  expectedOutcome: string;
+  validationMethod: string;
+}
+
+interface CoverageResult {
+  complete: boolean;
+  totalScenarios: number;
+  coveredScenarios: number;
+  missingScenarios: ValidationScenario[];
+  coverage: string; // "X/Y scenarios (Z%)"
+}
+
+/**
+ * Extract issue number from branch name
+ */
+function extractIssueNumberFromBranch(): number | null {
+  const branchName = process.env.GIT_BRANCH || '';
+  const match = branchName.match(/feature\/(\d+)/);
+  return match ? parseInt(match[1]) : null;
+}
+
+/**
+ * Find spec or design file for an issue
+ */
+function findSpecOrDesignFile(issueNumber: number): string | null {
+  const specDir = path.join(process.cwd(), 'docs', 'feature specs');
+  const rfcDir = path.join(process.cwd(), 'docs', 'rfcs');
+
+  // Try feature specs first
+  if (fs.existsSync(specDir)) {
+    const files = fs.readdirSync(specDir);
+    const specFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
+    if (specFile) {
+      return path.join(specDir, specFile);
+    }
+  }
+
+  // Then try RFCs
+  if (fs.existsSync(rfcDir)) {
+    const files = fs.readdirSync(rfcDir);
+    const rfcFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
+    if (rfcFile) {
+      return path.join(rfcDir, rfcFile);
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Extract validation plan table from markdown file
+ */
+function extractValidationPlan(filePath: string): ValidationScenario[] {
+  const content = fs.readFileSync(filePath, 'utf-8');
+  const scenarios: ValidationScenario[] = [];
+
+  // Find "## Validation Plan" section (not "Validation Plan Coverage")
+  const validationPlanMatch = content.match(/##\s+Validation Plan\s*\n[\s\S]*?(?=\n##|$)/i);
+  if (!validationPlanMatch) {
+    return scenarios;
+  }
+
+  const validationSection = validationPlanMatch[0];
+
+  // Look for markdown table - find table header row with "User Scenario"
+  // Pattern: | User Scenario | Expected outcome | Validation method |
+  const tablePattern = /\|[\s\*]*User Scenario[\s\*]*\|[\s\*]*Expected outcome[\s\*]*\|[\s\*]*Validation method[\s\*]*\|[\s]*\n\|[\s:\-|]+\|[\s]*\n((?:\|.*?\|[\s]*\n)+)/i;
+  const tableMatch = validationSection.match(tablePattern);
+
+  if (!tableMatch) {
+    return scenarios;
+  }
+
+  const tableRows = tableMatch[1].trim().split('\n').filter(row => row.trim().startsWith('|'));
+
+  for (const row of tableRows) {
+    // Skip separator rows (all dashes/pipes)
+    if (row.match(/^\|[\s:\-|]+\|$/)) {
+      continue;
+    }
+
+    // Parse table row: | Scenario | Outcome | Method |
+    // Handle both formats: | **bold** | text | and | text | text |
+    const cells = row.split('|').map(c => c.trim()).filter(c => c && c !== '');
+
+    if (cells.length >= 2) {
+      // Remove markdown bold (**text**) from cells
+      const cleanCell = (cell: string): string => {
+        return cell
+          .replace(/^\*\*/, '')
+          .replace(/\*\*$/, '')
+          .replace(/^\*\*/g, '')
+          .replace(/\*\*/g, '')
+          .trim();
+      };
+
+      const userScenario = cleanCell(cells[0] || '');
+      const expectedOutcome = cleanCell(cells[1] || '');
+      const validationMethod = cleanCell(cells[2] || 'Manual validation');
+
+      // Skip header rows
+      const scenarioLower = userScenario.toLowerCase();
+      const outcomeLower = expectedOutcome.toLowerCase();
+      if (scenarioLower.includes('user scenario') ||
+        (scenarioLower.includes('scenario') && outcomeLower.includes('expected')) ||
+        scenarioLower === 'user scenario' ||
+        outcomeLower === 'expected outcome') {
+        continue;
+      }
+
+      // Skip empty rows or separator rows
+      if (!userScenario || userScenario === '-' || userScenario.length === 0) {
+        continue;
+      }
+
+      scenarios.push({
+        userScenario,
+        expectedOutcome,
+        validationMethod
+      });
+    }
+  }
+
+  return scenarios;
+}
+
+/**
+ * Check if evidence exists for a scenario
+ * Evidence can be in PR comments, test.log, or other sources
+ */
+function checkEvidenceForScenario(scenario: ValidationScenario, issueNumber: number): boolean {
+  // Extract key terms from scenario for searching
+  const scenarioKeywords = scenario.userScenario
+    .toLowerCase()
+    .replace(/\*\*/g, '')
+    .split(/\s+/)
+    .filter(w => w.length > 3 && !['agent', 'the', 'with', 'without'].includes(w));
+
+  const outcomeKeywords = scenario.expectedOutcome
+    .toLowerCase()
+    .replace(/\*\*/g, '')
+    .split(/\s+/)
+    .filter(w => w.length > 3);
+
+  // Method 1: Check test.log for scenario keywords
+  const testLogPath = path.join(process.cwd(), 'test.log');
+  if (fs.existsSync(testLogPath)) {
+    const testLogContent = fs.readFileSync(testLogPath, 'utf-8').toLowerCase();
+    // Check if multiple keywords from scenario appear
+    const matchingKeywords = scenarioKeywords.filter(kw => testLogContent.includes(kw));
+    if (matchingKeywords.length >= 2) {
+      return true;
+    }
+  }
+
+  // Method 2: Check for evidence file (created by agent with scenario validation)
+  const evidenceFiles = ['test-evidence.md', 'validation-evidence.md', 'implementation-evidence.md'];
+  for (const evidenceFile of evidenceFiles) {
+    const evidencePath = path.join(process.cwd(), evidenceFile);
+    if (fs.existsSync(evidencePath)) {
+      const evidenceContent = fs.readFileSync(evidencePath, 'utf-8').toLowerCase();
+      const matchingKeywords = scenarioKeywords.filter(kw => evidenceContent.includes(kw));
+      if (matchingKeywords.length >= 2) {
+        return true;
+      }
+    }
+  }
+
+  // Method 3: Check server.log for scenario-related activity
+  const serverLogPath = path.join(process.cwd(), 'server.log');
+  if (fs.existsSync(serverLogPath)) {
+    const serverLogContent = fs.readFileSync(serverLogPath, 'utf-8').toLowerCase();
+    // Look for validation-related keywords
+    if (scenarioKeywords.some(kw => serverLogContent.includes(kw)) ||
+      outcomeKeywords.some(kw => serverLogContent.includes('gate') && serverLogContent.includes(kw))) {
+      return true;
+    }
+  }
+
+  // Note: PR comment checking is done in the shell script (Gate 9)
+  // This TypeScript tool focuses on file-based evidence
+
+  return false; // Conservative: require evidence
+}
+
+/**
+ * Main function
+ */
+function main(): void {
+  const issueNumberArg = process.argv[2];
+  const issueNumber = issueNumberArg ? parseInt(issueNumberArg) : extractIssueNumberFromBranch();
+
+  if (!issueNumber) {
+    console.error('❌ ERROR: Could not determine issue number');
+    console.error('   Usage: npx tsx validate-coverage.ts <issue-number>');
+    console.error('   Or set GIT_BRANCH environment variable with feature/N-* format');
+    process.exit(1);
+  }
+
+  const specFile = findSpecOrDesignFile(issueNumber);
+
+  if (!specFile) {
+    // No spec/design found - Gate 9 warns but doesn't block
+    console.log('ℹ️  No spec/design found for issue #' + issueNumber);
+    console.log('   Skipping validation plan coverage check');
+    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no spec/design)');
+    process.exit(0);
+  }
+
+  console.log(`Found spec/design: ${specFile}`);
+
+  const scenarios = extractValidationPlan(specFile);
+
+  if (scenarios.length === 0) {
+    console.log('ℹ️  No validation plan table found in spec/design');
+    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no validation plan)');
+    process.exit(0);
+  }
+
+  console.log(`Found ${scenarios.length} validation scenario(s)`);
+
+  // Check evidence for each scenario
+  const covered: ValidationScenario[] = [];
+  const missing: ValidationScenario[] = [];
+
+  for (const scenario of scenarios) {
+    if (checkEvidenceForScenario(scenario, issueNumber)) {
+      covered.push(scenario);
+    } else {
+      missing.push(scenario);
+    }
+  }
+
+  const coverage: CoverageResult = {
+    complete: missing.length === 0,
+    totalScenarios: scenarios.length,
+    coveredScenarios: covered.length,
+    missingScenarios: missing,
+    coverage: `${covered.length}/${scenarios.length} scenarios (${Math.round(covered.length / scenarios.length * 100)}%)`
+  };
+
+  // Output results
+  console.log('\n📊 Validation Plan Coverage:');
+  console.log(`   ${coverage.coverage}`);
+
+  if (coverage.complete) {
+    console.log('✅ PASSED: All validation scenarios have evidence');
+    console.log('   Gate 9: Validation Plan Coverage - PASSED');
+    process.exit(0);
+  } else {
+    console.log('❌ FAILED: Missing evidence for validation scenarios:');
+    for (const scenario of missing) {
+      console.log(`   - "${scenario.userScenario}"`);
+      console.log(`     Expected: ${scenario.expectedOutcome}`);
+      console.log(`     Method: ${scenario.validationMethod}`);
+    }
+    console.log('\n   Required: Provide evidence for all validation scenarios before marking PR ready');
+    console.log('   Evidence can be: PR comment, test output, manual verification log');
+    console.log('   Gate 9: Validation Plan Coverage - FAILED');
+    process.exit(1);
+  }
+}
+
+// Run main function
+main();