@hyperfixi/testing-framework 2.0.0

package/src/multilingual/cli.ts

@@ -0,0 +1,224 @@
+#!/usr/bin/env node
+/**
+ * Multilingual Testing CLI
+ *
+ * Command-line interface for running multilingual tests.
+ */
+
+import { TestOrchestrator } from './orchestrator';
+import { RegressionReporter } from './reporters/regression-reporter';
+import type { TestConfig, LanguageCode } from './types';
+
+/**
+ * Parse command-line arguments
+ */
+function parseArgs(): TestConfig {
+  const args = process.argv.slice(2);
+  const config: TestConfig = {
+    mode: 'quick',
+    quickModeLimit: 10,
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+
+    switch (arg) {
+      case '--language':
+      case '-l': {
+        const lang = args[++i];
+        if (lang) config.languages = [lang as LanguageCode];
+        break;
+      }
+
+      case '--languages': {
+        const langs = args[++i];
+        if (langs) config.languages = langs.split(',') as LanguageCode[];
+        break;
+      }
+
+      case '--bundle':
+      case '-b': {
+        const bundle = args[++i];
+        if (bundle) config.bundle = bundle;
+        break;
+      }
+
+      case '--build':
+        config.build = true;
+        break;
+
+      case '--mode':
+      case '-m':
+        config.mode = args[++i] as 'quick' | 'full';
+        break;
+
+      case '--quick':
+        config.mode = 'quick';
+        break;
+
+      case '--full':
+        config.mode = 'full';
+        break;
+
+      case '--verbose':
+      case '-v':
+        config.verbose = true;
+        break;
+
+      case '--regression':
+      case '-r':
+        config.regression = true;
+        break;
+
+      case '--confidence':
+      case '-c': {
+        const conf = args[++i];
+        if (conf) config.confidenceThreshold = parseFloat(conf);
+        break;
+      }
+
+      case '--verified-only':
+        config.verifiedOnly = true;
+        break;
+
+      case '--categories': {
+        const cats = args[++i];
+        if (cats) config.categories = cats.split(',');
+        break;
+      }
+
+      case '--limit': {
+        const lim = args[++i];
+        if (lim) config.quickModeLimit = parseInt(lim, 10);
+        break;
+      }
+
+      case '--help':
+      case '-h':
+        showHelp();
+        process.exit(0);
+        break;
+
+      case '--save-baseline':
+        // Special flag to save results as baseline
+        runAndSaveBaseline();
+        return config; // Early return to avoid running tests below
+
+      default:
+        if (arg && arg.startsWith('-')) {
+          console.error(`Unknown option: ${arg}`);
+          process.exit(1);
+        }
+        break;
+    }
+  }
+
+  return config;
+}
+
+/**
+ * Show help message
+ */
+function showHelp(): void {
+  console.log(`
+Multilingual Test Runner
+
+USAGE:
+  npm run test:multilingual [OPTIONS]
+
+OPTIONS:
+  -l, --language <code>        Test specific language (en, ja, es, etc.)
+      --languages <codes>      Test multiple languages (comma-separated)
+  -b, --bundle <name>          Use specific bundle
+      --build                  Build bundle before testing
+  -m, --mode <mode>            Test mode: 'quick' or 'full' (default: quick)
+      --quick                  Quick mode (10 patterns per language)
+      --full                   Full mode (all patterns)
+  -v, --verbose                Enable verbose output
+  -r, --regression             Compare to baseline and report regressions
+  -c, --confidence <n>         Minimum confidence threshold (0-1)
+      --verified-only          Only test verified translations
+      --categories <cats>      Filter by categories (comma-separated)
+      --limit <n>              Patterns per language in quick mode (default: 10)
+      --save-baseline          Save current results as new baseline
+  -h, --help                   Show this help message
+
+EXAMPLES:
+  # Test all languages in quick mode
+  npm run test:multilingual
+
+  # Test specific language with verbose output
+  npm run test:multilingual -- --language ja --verbose
+
+  # Test multiple languages in full mode
+  npm run test:multilingual -- --languages ja,ko,es --full
+
+  # Build bundle and test with regression comparison
+  npm run test:multilingual -- --build --regression
+
+  # Save current results as baseline
+  npm run test:multilingual -- --save-baseline
+`);
+}
+
+/**
+ * Run tests and save as baseline
+ */
+async function runAndSaveBaseline(): Promise<void> {
+  try {
+    // Parse config without --save-baseline
+    const args = process.argv.slice(2).filter(arg => arg !== '--save-baseline');
+    process.argv = [...process.argv.slice(0, 2), ...args];
+
+    const config = parseArgs();
+    config.regression = true; // Enable regression reporter
+
+    const orchestrator = new TestOrchestrator(config);
+    const results = await orchestrator.run();
+
+    // Save as baseline
+    const regressionReporter = orchestrator.getRegressionReporter();
+    if (regressionReporter) {
+      regressionReporter.saveAsBaseline(results);
+    } else {
+      // Create reporter and save
+      const reporter = new RegressionReporter();
+      reporter.saveAsBaseline(results);
+    }
+
+    process.exit(0);
+  } catch (error) {
+    console.error('Error:', error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+}
+
+/**
+ * Main CLI entry point
+ */
+async function main(): Promise<void> {
+  try {
+    const config = parseArgs();
+
+    const orchestrator = new TestOrchestrator(config);
+    const results = await orchestrator.run();
+
+    // Exit with appropriate code
+    const exitCode = results.summary.overallStatus === 'pass' ? 0 : 1;
+    process.exit(exitCode);
+  } catch (error) {
+    console.error('Error:', error instanceof Error ? error.message : String(error));
+    if (error instanceof Error && error.stack) {
+      console.error(error.stack);
+    }
+    process.exit(1);
+  }
+}
+
+// Check if running as CLI
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}
+
+// Export for programmatic use
+export { main, parseArgs, showHelp };

package/src/multilingual/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Multilingual Testing Framework
+ *
+ * Automated validation of HyperFixi's multilingual system.
+ */
+
+// Main orchestrator
+export { TestOrchestrator, runMultilingualTests } from './orchestrator';
+
+// Types
+export type {
+  LanguageCode,
+  WordOrder,
+  Pattern,
+  PatternTranslation,
+  TestConfig,
+  ParseResult,
+  SizeResult,
+  LanguageResults,
+  TestResults,
+  Baseline,
+  RegressionResult,
+  BundleInfo,
+  BundleBuildOptions,
+  Reporter,
+  Validator,
+  SamplingStrategy,
+} from './types';
+
+// Pattern loading
+export {
+  loadPatterns,
+  getPatternStatistics,
+  loadPatternsByCategory,
+  countPatternsByLanguage,
+} from './pattern-loader';
+
+// Bundle building
+export {
+  findBundleForLanguages,
+  getBundleInfo,
+  buildBundle,
+  selectBundle,
+  listAvailableBundles,
+  bundleExists,
+  getBundleSize,
+  estimateBundleSize,
+} from './bundle-builder';
+
+// Validators
+export {
+  ParseValidator,
+  validatePatterns,
+  validateAndSummarize,
+} from './validators/parse-validator';
+export {
+  SizeValidator,
+  validateBundleSize,
+  validateAndSummarizeSizes,
+} from './validators/size-validator';
+
+// Reporters
+export { ConsoleReporter, createConsoleReporter } from './reporters/console-reporter';
+export { JSONReporter, createJSONReporter } from './reporters/json-reporter';
+export { RegressionReporter, createRegressionReporter } from './reporters/regression-reporter';
+
+// CLI
+export { main as runCLI, parseArgs, showHelp } from './cli';

package/src/multilingual/orchestrator.ts

@@ -0,0 +1,275 @@
+/**
+ * Orchestrator - Main test runner that coordinates all components
+ */
+
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
+import { loadPatterns } from './pattern-loader';
+import { selectBundle, getBundleInfo } from './bundle-builder';
+import { ParseValidator } from './validators/parse-validator';
+import { SizeValidator } from './validators/size-validator';
+import { ConsoleReporter } from './reporters/console-reporter';
+import { JSONReporter } from './reporters/json-reporter';
+import { RegressionReporter } from './reporters/regression-reporter';
+import type { TestConfig, TestResults, LanguageResults, LanguageCode, Reporter } from './types';
+
+const execAsync = promisify(exec);
+
+/**
+ * Test Orchestrator
+ *
+ * Coordinates the entire multilingual test workflow:
+ * 1. Load patterns from database
+ * 2. Select/build bundles
+ * 3. Run validators
+ * 4. Report results
+ */
+export class TestOrchestrator {
+  private config: TestConfig;
+  private reporters: Reporter[] = [];
+
+  constructor(config: TestConfig) {
+    this.config = config;
+    this.setupReporters();
+  }
+
+  /**
+   * Setup default reporters
+   */
+  private setupReporters(): void {
+    // Always add console reporter
+    this.reporters.push(
+      new ConsoleReporter({
+        verbose: this.config.verbose || false,
+        useColors: true,
+      })
+    );
+
+    // Add JSON reporter for structured output
+    this.reporters.push(new JSONReporter('./test-results/results.json'));
+
+    // Add regression reporter if requested
+    if (this.config.regression) {
+      this.reporters.push(new RegressionReporter('./test-results/baseline.json'));
+    }
+  }
+
+  /**
+   * Add custom reporter
+   */
+  addReporter(reporter: Reporter): void {
+    this.reporters.push(reporter);
+  }
+
+  /**
+   * Run the test suite
+   */
+  async run(): Promise<TestResults> {
+    // Notify reporters of test start
+    for (const reporter of this.reporters) {
+      reporter.reportStart(this.config);
+    }
+
+    try {
+      // Get git commit for tracking
+      const commit = await this.getGitCommit();
+
+      // Load patterns
+      const patterns = await loadPatterns(this.config);
+
+      // Group patterns by language
+      const patternsByLanguage = this.groupByLanguage(patterns);
+
+      // Test each language
+      const languageResults: LanguageResults[] = [];
+
+      for (const [language, langPatterns] of patternsByLanguage.entries()) {
+        const result = await this.testLanguage(language as LanguageCode, langPatterns);
+        languageResults.push(result);
+      }
+
+      // Collect bundle information
+      const bundles: TestResults['bundles'] = {};
+      for (const langResult of languageResults) {
+        if (!bundles[langResult.bundle]) {
+          const bundleInfo = await getBundleInfo(langResult.bundle);
+          if (bundleInfo && bundleInfo.exists) {
+            bundles[langResult.bundle] = {
+              size: bundleInfo.size,
+              languages: bundleInfo.languages,
+            };
+          }
+        }
+      }
+
+      // Calculate summary
+      const summary = {
+        totalPatterns: patterns.length,
+        totalSuccess: languageResults.reduce((sum, r) => sum + r.parseSuccess, 0),
+        totalFailure: languageResults.reduce((sum, r) => sum + r.parseFailure, 0),
+        totalDuration: languageResults.reduce((sum, r) => sum + r.duration, 0),
+        overallStatus: this.calculateOverallStatus(languageResults),
+      };
+
+      const results: TestResults = {
+        timestamp: new Date().toISOString(),
+        commit,
+        config: this.config,
+        languageResults,
+        bundles,
+        summary,
+      };
+
+      // Report completion
+      for (const reporter of this.reporters) {
+        reporter.reportComplete(results);
+      }
+
+      // Report regression if enabled
+      if (this.config.regression) {
+        const regressionReporter = this.reporters.find(r => r instanceof RegressionReporter) as
+          | RegressionReporter
+          | undefined;
+
+        if (regressionReporter?.hasBaseline()) {
+          const regressionResults = regressionReporter.getRegressionResults();
+          for (const reporter of this.reporters) {
+            if (reporter.reportRegression) {
+              reporter.reportRegression(regressionResults);
+            }
+          }
+        }
+      }
+
+      return results;
+    } catch (error) {
+      // Report error
+      const err = error instanceof Error ? error : new Error(String(error));
+      for (const reporter of this.reporters) {
+        reporter.reportError(err);
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * Test a single language
+   */
+  private async testLanguage(language: LanguageCode, patterns: any[]): Promise<LanguageResults> {
+    const startTime = performance.now();
+
+    // Select bundle for this language
+    const bundle = this.config.bundle
+      ? await getBundleInfo(this.config.bundle)
+      : await selectBundle([language], this.config.build || false);
+
+    if (!bundle || !bundle.exists) {
+      throw new Error(`Bundle not found for language: ${language}`);
+    }
+
+    // Notify reporters
+    for (const reporter of this.reporters) {
+      reporter.reportLanguageStart(language, bundle.name);
+    }
+
+    // Run parse validation
+    const parseValidator = new ParseValidator();
+    const parseResults = await parseValidator.validate(patterns);
+
+    // Calculate statistics
+    const parseSuccess = parseResults.filter(r => r.success).length;
+    const parseFailure = parseResults.length - parseSuccess;
+    const parseRate = parseResults.length > 0 ? parseSuccess / parseResults.length : 0;
+    const avgConfidence =
+      parseResults
+        .filter(r => r.success && r.confidence !== undefined)
+        .reduce((sum, r) => sum + (r.confidence || 0), 0) / (parseSuccess || 1);
+
+    // Determine status
+    const status = parseRate >= 0.95 ? 'pass' : parseRate >= 0.8 ? 'warning' : 'fail';
+
+    const result: LanguageResults = {
+      language,
+      bundle: bundle.name,
+      bundleSize: bundle.size,
+      parseResults,
+      parseSuccess,
+      parseFailure,
+      parseRate,
+      avgConfidence,
+      duration: performance.now() - startTime,
+      status,
+    };
+
+    // Notify reporters
+    for (const reporter of this.reporters) {
+      reporter.reportLanguageComplete(result);
+    }
+
+    return result;
+  }
+
+  /**
+   * Group patterns by language
+   */
+  private groupByLanguage(patterns: any[]): Map<string, any[]> {
+    const grouped = new Map<string, any[]>();
+
+    for (const pattern of patterns) {
+      const lang = pattern.language;
+      if (!grouped.has(lang)) {
+        grouped.set(lang, []);
+      }
+      grouped.get(lang)!.push(pattern);
+    }
+
+    return grouped;
+  }
+
+  /**
+   * Calculate overall test status
+   */
+  private calculateOverallStatus(results: LanguageResults[]): 'pass' | 'warning' | 'fail' {
+    const hasFail = results.some(r => r.status === 'fail');
+    const hasWarning = results.some(r => r.status === 'warning');
+
+    if (hasFail) return 'fail';
+    if (hasWarning) return 'warning';
+    return 'pass';
+  }
+
+  /**
+   * Get current git commit
+   */
+  private async getGitCommit(): Promise<string | undefined> {
+    try {
+      const { stdout } = await execAsync('git rev-parse --short HEAD');
+      return stdout.trim();
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Get regression reporter
+   */
+  getRegressionReporter(): RegressionReporter | undefined {
+    return this.reporters.find(r => r instanceof RegressionReporter) as
+      | RegressionReporter
+      | undefined;
+  }
+}
+
+/**
+ * Quick run helper
+ */
+export async function runMultilingualTests(config: Partial<TestConfig>): Promise<TestResults> {
+  const fullConfig: TestConfig = {
+    mode: 'quick',
+    quickModeLimit: 10,
+    ...config,
+  };
+
+  const orchestrator = new TestOrchestrator(fullConfig);
+  return await orchestrator.run();
+}