@hyperfixi/testing-framework 2.0.0

package/src/multilingual/reporters/regression-reporter.ts

@@ -0,0 +1,263 @@
+/**
+ * Regression Reporter - Compares current results against baseline
+ */
+
+import { readFileSync, existsSync, writeFileSync } from 'node:fs';
+import type {
+  Reporter,
+  TestConfig,
+  LanguageCode,
+  LanguageResults,
+  TestResults,
+  Baseline,
+  RegressionResult,
+} from '../types';
+
+/**
+ * Regression Reporter
+ *
+ * Compares test results against a baseline and reports regressions.
+ */
+export class RegressionReporter implements Reporter {
+  private baselinePath: string;
+  private baseline: Baseline | null = null;
+  private currentResults: TestResults | null = null;
+  private regressionResults: RegressionResult[] = [];
+
+  constructor(baselinePath: string = './test-results/baseline.json') {
+    this.baselinePath = baselinePath;
+    this.loadBaseline();
+  }
+
+  /**
+   * Load baseline from file
+   */
+  private loadBaseline(): void {
+    if (!existsSync(this.baselinePath)) {
+      console.warn(`Warning: Baseline file not found at ${this.baselinePath}`);
+      return;
+    }
+
+    try {
+      const data = readFileSync(this.baselinePath, 'utf-8');
+      this.baseline = JSON.parse(data);
+    } catch (error) {
+      console.error(
+        `Error loading baseline: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Report test start
+   */
+  reportStart(_config: TestConfig): void {
+    // No-op
+  }
+
+  /**
+   * Report language start
+   */
+  reportLanguageStart(_language: LanguageCode, _bundle: string): void {
+    // No-op
+  }
+
+  /**
+   * Report language complete
+   */
+  reportLanguageComplete(_results: LanguageResults): void {
+    // No-op
+  }
+
+  /**
+   * Report complete test run - compare to baseline
+   */
+  reportComplete(results: TestResults): void {
+    this.currentResults = results;
+
+    if (!this.baseline) {
+      console.log('No baseline available - skipping regression analysis');
+      return;
+    }
+
+    this.regressionResults = this.compareToBaseline(results);
+  }
+
+  /**
+   * Get regression results
+   */
+  getRegressionResults(): RegressionResult[] {
+    return this.regressionResults;
+  }
+
+  /**
+   * Report error
+   */
+  reportError(_error: Error): void {
+    // No-op
+  }
+
+  /**
+   * Compare current results to baseline
+   */
+  private compareToBaseline(results: TestResults): RegressionResult[] {
+    if (!this.baseline) return [];
+
+    const regressions: RegressionResult[] = [];
+
+    for (const langResult of results.languageResults) {
+      const baselineLang = this.baseline.languages[langResult.language];
+      if (!baselineLang) continue;
+
+      const parseRateDelta = (langResult.parseRate - baselineLang.parseRate) * 100;
+      const avgConfidenceDelta = langResult.avgConfidence - baselineLang.avgConfidence;
+      const bundleSizeDelta = this.getBundleSizeDelta(langResult, baselineLang);
+
+      // Identify new failures and successes
+      const newFailures = this.findNewFailures(langResult, baselineLang);
+      const newSuccesses = this.findNewSuccesses(langResult, baselineLang);
+
+      // Determine status
+      let status: 'improved' | 'regressed' | 'unchanged' = 'unchanged';
+      if (parseRateDelta < -5 || newFailures.length > 0) {
+        status = 'regressed';
+      } else if (parseRateDelta > 5 || newSuccesses.length > 0) {
+        status = 'improved';
+      }
+
+      regressions.push({
+        language: langResult.language,
+        parseRateDelta,
+        avgConfidenceDelta,
+        bundleSizeDelta: bundleSizeDelta !== undefined ? bundleSizeDelta : undefined,
+        newFailures,
+        newSuccesses,
+        status,
+      });
+    }
+
+    return regressions;
+  }
+
+  /**
+   * Get bundle size delta
+   */
+  private getBundleSizeDelta(
+    current: LanguageResults,
+    baseline: { bundleSize: number | undefined }
+  ): number | undefined {
+    if (!current.bundleSize || !baseline.bundleSize) return undefined;
+    return ((current.bundleSize - baseline.bundleSize) / baseline.bundleSize) * 100;
+  }
+
+  /**
+   * Find patterns that failed in current but passed in baseline
+   */
+  private findNewFailures(
+    current: LanguageResults,
+    baseline: {
+      parseSuccess: number;
+      patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
+    }
+  ): string[] {
+    // If no pattern-level baseline, return empty array
+    if (!baseline.patterns) return [];
+
+    const newFailures: string[] = [];
+
+    for (const result of current.parseResults) {
+      const baselinePattern = baseline.patterns[result.pattern.codeExampleId];
+      if (baselinePattern && baselinePattern.success && !result.success) {
+        newFailures.push(result.pattern.codeExampleId);
+      }
+    }
+
+    return newFailures.slice(0, 10); // Limit to 10 for reporting
+  }
+
+  /**
+   * Find patterns that passed in current but failed in baseline
+   */
+  private findNewSuccesses(
+    current: LanguageResults,
+    baseline: {
+      parseSuccess: number;
+      patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
+    }
+  ): string[] {
+    // If no pattern-level baseline, return empty array
+    if (!baseline.patterns) return [];
+
+    const newSuccesses: string[] = [];
+
+    for (const result of current.parseResults) {
+      const baselinePattern = baseline.patterns[result.pattern.codeExampleId];
+      if (baselinePattern && !baselinePattern.success && result.success) {
+        newSuccesses.push(result.pattern.codeExampleId);
+      }
+    }
+
+    return newSuccesses.slice(0, 10); // Limit to 10 for reporting
+  }
+
+  /**
+   * Save current results as new baseline
+   */
+  saveAsBaseline(results: TestResults): void {
+    const baseline: Baseline = {
+      timestamp: results.timestamp,
+      commit: results.commit || 'unknown',
+      languages: {},
+      bundles: {} as Record<
+        string,
+        { size: number; languages: LanguageCode[]; gzipSize: number | undefined }
+      >,
+    };
+
+    // Convert bundles to proper format
+    for (const [name, info] of Object.entries(results.bundles)) {
+      baseline.bundles[name] = {
+        size: info.size,
+        languages: info.languages,
+        gzipSize: info.gzipSize ?? undefined,
+      };
+    }
+
+    for (const langResult of results.languageResults) {
+      // Build pattern-level tracking
+      const patterns: Record<string, { success: boolean; confidence: number | undefined }> = {};
+      for (const result of langResult.parseResults) {
+        patterns[result.pattern.codeExampleId] = {
+          success: result.success,
+          confidence: result.confidence,
+        };
+      }
+
+      baseline.languages[langResult.language] = {
+        parseSuccess: langResult.parseSuccess,
+        parseFailure: langResult.parseFailure,
+        parseRate: langResult.parseRate,
+        avgConfidence: langResult.avgConfidence,
+        bundleSize: langResult.bundleSize ?? undefined,
+        patterns,
+      };
+    }
+
+    writeFileSync(this.baselinePath, JSON.stringify(baseline, null, 2));
+    console.log(`Baseline saved to ${this.baselinePath}`);
+  }
+
+  /**
+   * Check if baseline exists
+   */
+  hasBaseline(): boolean {
+    return this.baseline !== null;
+  }
+}
+
+/**
+ * Create a regression reporter
+ */
+export function createRegressionReporter(baselinePath?: string): RegressionReporter {
+  return new RegressionReporter(baselinePath);
+}

package/src/multilingual/tools/analyze-failures.ts

@@ -0,0 +1,326 @@
+#!/usr/bin/env node
+/**
+ * Parse Failure Analysis Tool
+ *
+ * Analyzes parse failures to identify patterns and suggest fixes
+ */
+
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { FailureAnalyzer } from '../validators/parse-diagnostics';
+import type { TestResults, ParseResult, LanguageCode } from '../types';
+
+interface AnalysisOptions {
+  language: LanguageCode | undefined;
+  verbose: boolean;
+  outputFile: string | undefined;
+}
+
+/**
+ * Parse command-line arguments
+ */
+function parseArgs(): AnalysisOptions {
+  const args = process.argv.slice(2);
+  const options: AnalysisOptions = {
+    language: undefined,
+    verbose: false,
+    outputFile: undefined,
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case '--language':
+      case '-l': {
+        const lang = args[++i];
+        if (lang) options.language = lang as LanguageCode;
+        break;
+      }
+
+      case '--verbose':
+      case '-v':
+        options.verbose = true;
+        break;
+
+      case '--output':
+      case '-o': {
+        const file = args[++i];
+        if (file) options.outputFile = file;
+        break;
+      }
+
+      case '--help':
+      case '-h':
+        showHelp();
+        process.exit(0);
+        break;
+    }
+  }
+
+  return options;
+}
+
+/**
+ * Show help message
+ */
+function showHelp(): void {
+  console.log(`
+Parse Failure Analysis Tool
+
+Analyzes test results to identify failure patterns and suggest improvements.
+
+USAGE:
+  npm run analyze-failures [OPTIONS]
+
+OPTIONS:
+  -l, --language <code>    Analyze specific language only
+  -v, --verbose            Show detailed output
+  -o, --output <file>      Write report to file
+  -h, --help               Show this help message
+
+EXAMPLES:
+  # Analyze all failures
+  npm run analyze-failures
+
+  # Analyze Japanese failures with verbose output
+  npm run analyze-failures -- --language ja --verbose
+
+  # Generate report file
+  npm run analyze-failures -- --output failure-analysis.md
+`);
+}
+
+/**
+ * Load test results
+ */
+function loadResults(): TestResults | undefined {
+  const resultsPath = join(process.cwd(), 'test-results', 'results.json');
+
+  if (!existsSync(resultsPath)) {
+    console.error('Error: No test results found at:', resultsPath);
+    console.error('Run tests first: npm run test:multilingual -- --full');
+    return undefined;
+  }
+
+  const data = readFileSync(resultsPath, 'utf-8');
+  return JSON.parse(data);
+}
+
+/**
+ * Extract all parse results from test results
+ */
+function extractParseResults(
+  results: TestResults,
+  language: LanguageCode | undefined
+): ParseResult[] {
+  let allResults: ParseResult[] = [];
+
+  for (const langResult of results.languageResults) {
+    if (language && langResult.language !== language) continue;
+    allResults = allResults.concat(langResult.parseResults);
+  }
+
+  return allResults;
+}
+
+/**
+ * Generate detailed failure breakdown
+ */
+function generateDetailedBreakdown(analyzer: FailureAnalyzer, results: ParseResult[]): string {
+  const failures = results.filter(r => !r.success);
+  const byLanguage = analyzer.analyzeByLanguage(results);
+
+  let output = '\n';
+  output += '═══════════════════════════════════════════════════════════════\n';
+  output += '  DETAILED FAILURE BREAKDOWN\n';
+  output += '═══════════════════════════════════════════════════════════════\n\n';
+
+  const sortedLangs = Object.entries(byLanguage)
+    .sort(([, a], [, b]) => b.failures - a.failures)
+    .filter(([, data]) => data.failures > 0);
+
+  for (const [lang, data] of sortedLangs) {
+    const failRate = ((data.failures / data.totalPatterns) * 100).toFixed(1);
+    const passRate = (100 - parseFloat(failRate)).toFixed(1);
+
+    output += `\n${lang.toUpperCase()}\n`;
+    output += '─'.repeat(60) + '\n';
+    output += `  Status: ${data.failures}/${data.totalPatterns} failures (${failRate}% fail, ${passRate}% pass)\n\n`;
+
+    output += '  Common Error Types:\n';
+    const errorCounts: Map<string, number> = new Map();
+    const langFailures = failures.filter(f => f.pattern.language === lang);
+
+    for (const failure of langFailures) {
+      const err = failure.error || 'Unknown';
+      errorCounts.set(err, (errorCounts.get(err) || 0) + 1);
+    }
+
+    const sortedErrors = Array.from(errorCounts.entries())
+      .sort(([, a], [, b]) => b - a)
+      .slice(0, 5);
+
+    for (const [error, count] of sortedErrors) {
+      const pct = ((count / data.failures) * 100).toFixed(1);
+      output += `    - ${error}: ${count} (${pct}%)\n`;
+    }
+
+    output += '\n  Problematic Patterns (first 5):\n';
+    data.problematicPatterns.slice(0, 5).forEach((pattern: string, i: number) => {
+      output += `    ${i + 1}. "${pattern}"\n`;
+    });
+
+    output += '\n';
+  }
+
+  return output;
+}
+
+/**
+ * Generate recommendations
+ */
+function generateRecommendations(analyzer: FailureAnalyzer, results: ParseResult[]): string {
+  const failures = results.filter(r => !r.success);
+  const patterns = analyzer.identifyFailurePatterns(failures);
+  const byLanguage = analyzer.analyzeByLanguage(results);
+
+  let output = '\n';
+  output += '═══════════════════════════════════════════════════════════════\n';
+  output += '  RECOMMENDATIONS\n';
+  output += '═══════════════════════════════════════════════════════════════\n\n';
+
+  // Language-specific recommendations
+  const sovLanguages: LanguageCode[] = ['ja', 'ko', 'tr'];
+  const vsoLanguages: LanguageCode[] = ['ar'];
+  const sovFailures = sovLanguages.filter(
+    lang => byLanguage[lang] && byLanguage[lang].failures > byLanguage[lang].totalPatterns * 0.3
+  );
+  const vsoFailures = vsoLanguages.filter(
+    lang => byLanguage[lang] && byLanguage[lang].failures > byLanguage[lang].totalPatterns * 0.3
+  );
+
+  if (sovFailures.length > 0) {
+    output +=
+      '## SOV Language Issues (' + sovFailures.map(l => l.toUpperCase()).join(', ') + ')\n\n';
+    output += '  Priority Actions:\n';
+    output += '  1. Improve particle detection and positioning\n';
+    output += '  2. Enhance verb conjugation recognition\n';
+    output += '  3. Add support for complex clause patterns\n\n';
+    output += '  Implementation Files:\n';
+    output += '  - packages/semantic/src/tokenizers/ja.ts (Japanese)\n';
+    output += '  - packages/semantic/src/tokenizers/ko.ts (Korean)\n';
+    output += '  - packages/semantic/src/tokenizers/tr.ts (Turkish)\n';
+    output += '  - packages/semantic/src/parser/semantic-parser.ts\n\n';
+  }
+
+  if (vsoFailures.length > 0) {
+    output +=
+      '## VSO Language Issues (' + vsoFailures.map(l => l.toUpperCase()).join(', ') + ')\n\n';
+    output += '  Priority Actions:\n';
+    output += '  1. Implement verb-first pattern matching\n';
+    output += '  2. Enhance preposition handling\n';
+    output += '  3. Improve right-to-left text processing\n\n';
+    output += '  Implementation Files:\n';
+    output += '  - packages/semantic/src/tokenizers/ar.ts\n';
+    output += '  - packages/semantic/src/parser/semantic-parser.ts\n\n';
+  }
+
+  // Pattern-specific recommendations
+  output += '## Common Failure Patterns\n\n';
+
+  if (patterns.multiClause > failures.length * 0.2) {
+    output += `  ⚠️ Multi-clause patterns: ${patterns.multiClause} failures\n`;
+    output += '    → Improve conjunction handling (それから/then/ثم/그러면)\n';
+    output += '    → Better clause separation logic\n\n';
+  }
+
+  if (patterns.styleProperties > failures.length * 0.15) {
+    output += `  ⚠️ Style property access: ${patterns.styleProperties} failures\n`;
+    output += '    → Add *property syntax support across all languages\n';
+    output += '    → Validate CSS property name recognition\n\n';
+  }
+
+  if (patterns.eventModifiers > failures.length * 0.15) {
+    output += `  ⚠️ Event modifiers: ${patterns.eventModifiers} failures\n`;
+    output += '    → Improve bracket syntax parsing\n';
+    output += '    → Add debounced/throttled keyword support\n\n';
+  }
+
+  if (patterns.conditionals > failures.length * 0.15) {
+    output += `  ⚠️ Conditionals: ${patterns.conditionals} failures\n`;
+    output += '    → Enhance if/else block parsing\n';
+    output += '    → Add match/conditional keyword recognition\n\n';
+  }
+
+  if (patterns.windowEvents > failures.length * 0.1) {
+    output += `  ⚠️ Window events: ${patterns.windowEvents} failures\n`;
+    output += '    → Add window/document scope handling\n';
+    output += '    → Validate event target resolution\n\n';
+  }
+
+  if (patterns.complexSelectors > failures.length * 0.1) {
+    output += `  ⚠️ Complex selectors: ${patterns.complexSelectors} failures\n`;
+    output += '    → Improve closest/first/parent selector parsing\n';
+    output += '    → Add HTML literal syntax support (<div.class/>)\n\n';
+  }
+
+  return output;
+}
+
+/**
+ * Main analysis function
+ */
+async function main(): Promise<void> {
+  const options = parseArgs();
+
+  console.log('Loading test results...');
+  const results = loadResults();
+  if (!results) {
+    process.exit(1);
+  }
+
+  console.log('Analyzing failures...\n');
+  const parseResults = extractParseResults(results, options.language);
+  const analyzer = new FailureAnalyzer();
+
+  // Generate basic report
+  const report = analyzer.generateReport(parseResults);
+
+  // Generate detailed breakdown
+  const breakdown = generateDetailedBreakdown(analyzer, parseResults);
+
+  // Generate recommendations
+  const recommendations = generateRecommendations(analyzer, parseResults);
+
+  // Combine all sections
+  const fullReport = report + breakdown + recommendations;
+
+  // Output
+  if (options.outputFile) {
+    writeFileSync(options.outputFile, fullReport);
+    console.log(`Report written to: ${options.outputFile}`);
+  } else {
+    console.log(fullReport);
+  }
+
+  // Summary
+  const failures = parseResults.filter(r => !r.success);
+  const failRate = ((failures.length / parseResults.length) * 100).toFixed(1);
+
+  console.log('\n═══════════════════════════════════════════════════════════════');
+  console.log('  ANALYSIS COMPLETE');
+  console.log('═══════════════════════════════════════════════════════════════\n');
+  console.log(`  Analyzed: ${parseResults.length} patterns`);
+  console.log(`  Failures: ${failures.length} (${failRate}%)`);
+  console.log(`  Languages: ${options.language || 'all'}`);
+  console.log();
+}
+
+// Run if called directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch(err => {
+    console.error('Error:', err.message);
+    process.exit(1);
+  });
+}
+
+export { main, parseArgs, showHelp };