@hyperfixi/testing-framework 2.0.0

package/src/multilingual/pattern-loader.ts

@@ -0,0 +1,176 @@
+/**
+ * Pattern Loader - Queries patterns database for test cases
+ */
+
+import {
+  getAllPatterns,
+  getTranslationsByLanguage,
+  getVerifiedTranslations,
+  getHighConfidenceTranslations,
+  getPatternStats,
+  type Pattern,
+  type Translation,
+} from '@hyperfixi/patterns-reference';
+import type { LanguageCode, PatternTranslation, TestConfig, SamplingStrategy } from './types';
+
+/**
+ * Load patterns for testing based on configuration
+ */
+export async function loadPatterns(config: TestConfig): Promise<PatternTranslation[]> {
+  const languages = config.languages || (await getAllLanguages());
+  const results: PatternTranslation[] = [];
+
+  for (const language of languages) {
+    const translations = await loadTranslationsForLanguage(language, config);
+    results.push(...translations);
+  }
+
+  // Apply sampling if in quick mode
+  if (config.mode === 'quick') {
+    const limit = config.quickModeLimit || 10;
+    return samplePatterns(results, { type: 'stratified', perCategory: limit });
+  }
+
+  return results;
+}
+
+/**
+ * Load translations for a specific language
+ */
+async function loadTranslationsForLanguage(
+  language: LanguageCode,
+  config: TestConfig
+): Promise<PatternTranslation[]> {
+  let translations: Translation[];
+
+  // Query based on configuration
+  if (config.verifiedOnly) {
+    translations = await getVerifiedTranslations(language, 1000);
+  } else if (config.confidenceThreshold !== undefined) {
+    // Get high confidence for all languages, then filter
+    const allTranslations = await getHighConfidenceTranslations(config.confidenceThreshold, 1000);
+    translations = allTranslations.filter(t => t.language === language);
+  } else {
+    translations = await getTranslationsByLanguage(language, 1000);
+  }
+
+  // Convert to PatternTranslation format
+  return translations.map(t => mapToPatternTranslation(t));
+}
+
+/**
+ * Map Translation to PatternTranslation
+ */
+function mapToPatternTranslation(translation: Translation): PatternTranslation {
+  return {
+    codeExampleId: translation.codeExampleId,
+    language: translation.language as LanguageCode,
+    hyperscript: translation.hyperscript,
+    wordOrder: translation.wordOrder,
+    confidence: translation.confidence,
+    verifiedParses: translation.verifiedParses,
+    roleAlignmentScore: ((translation as any).roleAlignmentScore ?? 0) as number,
+  };
+}
+
+/**
+ * Get all supported languages from database
+ */
+async function getAllLanguages(): Promise<LanguageCode[]> {
+  const stats = await getPatternStats();
+  return Object.keys(stats.byLanguage) as LanguageCode[];
+}
+
+/**
+ * Sample patterns based on strategy
+ */
+function samplePatterns(
+  patterns: PatternTranslation[],
+  strategy: SamplingStrategy
+): PatternTranslation[] {
+  switch (strategy.type) {
+    case 'all':
+      return patterns;
+
+    case 'first':
+      return patterns.slice(0, strategy.count);
+
+    case 'random':
+      return shuffleArray([...patterns]).slice(0, strategy.count);
+
+    case 'stratified': {
+      // Group by language and sample from each
+      const byLanguage = new Map<LanguageCode, PatternTranslation[]>();
+      for (const pattern of patterns) {
+        const lang = pattern.language;
+        if (!byLanguage.has(lang)) {
+          byLanguage.set(lang, []);
+        }
+        byLanguage.get(lang)!.push(pattern);
+      }
+
+      const sampled: PatternTranslation[] = [];
+      for (const langPatterns of byLanguage.values()) {
+        sampled.push(...langPatterns.slice(0, strategy.perCategory));
+      }
+      return sampled;
+    }
+
+    default:
+      return patterns;
+  }
+}
+
+/**
+ * Fisher-Yates shuffle
+ */
+function shuffleArray<T>(array: T[]): T[] {
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    const temp = array[i];
+    array[i] = array[j] as T;
+    array[j] = temp as T;
+  }
+  return array;
+}
+
+/**
+ * Get pattern statistics for reporting
+ */
+export async function getPatternStatistics() {
+  return await getPatternStats();
+}
+
+/**
+ * Load patterns by category
+ */
+export async function loadPatternsByCategory(
+  categories: string[],
+  languages: LanguageCode[]
+): Promise<PatternTranslation[]> {
+  const results: PatternTranslation[] = [];
+
+  for (const language of languages) {
+    const translations = await getTranslationsByLanguage(language, 1000);
+
+    // Filter by category (would need to join with patterns table in real implementation)
+    // For now, return all translations
+    results.push(...translations.map(t => mapToPatternTranslation(t)));
+  }
+
+  return results;
+}
+
+/**
+ * Count patterns by language
+ */
+export async function countPatternsByLanguage(): Promise<Record<LanguageCode, number>> {
+  const stats = await getPatternStats();
+  const counts: Record<string, number> = {};
+
+  for (const [lang, data] of Object.entries(stats.byLanguage)) {
+    counts[lang] = data.count;
+  }
+
+  return counts as Record<LanguageCode, number>;
+}

package/src/multilingual/reporters/console-reporter.ts

@@ -0,0 +1,294 @@
+/**
+ * Console Reporter - Minimal, helpful console output
+ */
+
+import type {
+  Reporter,
+  TestConfig,
+  LanguageCode,
+  LanguageResults,
+  TestResults,
+  RegressionResult,
+} from '../types';
+
+/**
+ * ANSI color codes for terminal output
+ */
+const colors = {
+  reset: '\x1b[0m',
+  bright: '\x1b[1m',
+  dim: '\x1b[2m',
+
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  red: '\x1b[31m',
+  cyan: '\x1b[36m',
+  gray: '\x1b[90m',
+
+  bgGreen: '\x1b[42m',
+  bgYellow: '\x1b[43m',
+  bgRed: '\x1b[41m',
+};
+
+/**
+ * Console Reporter
+ *
+ * Outputs minimal, actionable test results to the console.
+ */
+export class ConsoleReporter implements Reporter {
+  private verbose: boolean;
+  private useColors: boolean;
+
+  constructor(options: { verbose?: boolean; useColors?: boolean } = {}) {
+    this.verbose = options.verbose ?? false;
+    this.useColors = options.useColors ?? true;
+  }
+
+  /**
+   * Report test start
+   */
+  reportStart(config: TestConfig): void {
+    this.log('\n' + this.header('Multilingual Test Runner v1.0.0'));
+    this.log(this.separator());
+    this.log('');
+
+    // Configuration summary
+    const languages = config.languages?.join(', ') || 'all';
+    const mode =
+      config.mode === 'quick'
+        ? `quick mode (${config.quickModeLimit || 10} patterns/lang)`
+        : 'full mode';
+
+    this.log(`${this.dim('Languages:')} ${languages} (${config.languages?.length || 'all'})`);
+    if (config.bundle) {
+      this.log(`${this.dim('Bundle:')} ${config.bundle}`);
+    }
+    this.log(`${this.dim('Mode:')} ${mode}`);
+    if (config.confidenceThreshold) {
+      this.log(`${this.dim('Min Confidence:')} ${config.confidenceThreshold}`);
+    }
+    this.log('');
+  }
+
+  /**
+   * Report language test start
+   */
+  reportLanguageStart(language: LanguageCode, bundle: string): void {
+    if (this.verbose) {
+      this.log(
+        `${this.cyan('→')} Testing ${this.bright(language.toUpperCase())} with ${bundle}...`
+      );
+    }
+  }
+
+  /**
+   * Report language test complete
+   */
+  reportLanguageComplete(results: LanguageResults): void {
+    const status =
+      results.status === 'pass'
+        ? this.green('✓')
+        : results.status === 'warning'
+          ? this.yellow('⚠')
+          : this.red('✗');
+
+    const rate = `${results.parseSuccess}/${results.parseSuccess + results.parseFailure}`;
+    const percentage = `(${(results.parseRate * 100).toFixed(0)}%)`;
+    const duration = `${(results.duration / 1000).toFixed(1)}s`;
+    const confidence = results.avgConfidence > 0 ? `conf: ${results.avgConfidence.toFixed(2)}` : '';
+
+    this.log(
+      `  ${status} ${this.bright(results.language.toUpperCase())}: ` +
+        `${rate} ${percentage}  ` +
+        `${this.dim('⏱')}  ${duration}` +
+        (confidence ? `  ${this.dim(confidence)}` : '')
+    );
+
+    // Show failures if verbose
+    if (this.verbose && results.parseFailure > 0) {
+      const failures = results.parseResults.filter(r => !r.success);
+      this.log(`    ${this.dim(`Failed patterns (${failures.length}):`)}`);
+
+      // Show all failures with pattern ID for tracking
+      for (const failure of failures) {
+        this.log(`    ${this.dim('✗')} ${failure.pattern.codeExampleId || 'unknown'}`);
+        this.log(`      ${this.dim('Input:')} ${failure.pattern.hyperscript}`);
+        if (failure.error) {
+          this.log(`      ${this.red('Error:')} ${failure.error}`);
+        }
+        if (failure.confidence !== undefined) {
+          this.log(`      ${this.dim('Confidence:')} ${failure.confidence.toFixed(2)}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Report complete test run
+   */
+  reportComplete(results: TestResults): void {
+    this.log('');
+    this.log(this.separator());
+
+    const statusIcon =
+      results.summary.overallStatus === 'pass'
+        ? this.green('✓ PASS')
+        : results.summary.overallStatus === 'warning'
+          ? this.yellow('⚠ WARNING')
+          : this.red('✗ FAIL');
+
+    const summary = `${results.summary.totalSuccess}/${results.summary.totalPatterns}`;
+    const duration = `${(results.summary.totalDuration / 1000).toFixed(1)}s`;
+
+    this.log(
+      `${this.bright('Summary:')} ${statusIcon}  ` +
+        `${summary}  ` +
+        `${this.dim('Duration:')} ${duration}`
+    );
+
+    // Show bundle sizes if available
+    if (Object.keys(results.bundles).length > 0) {
+      this.log('');
+      this.log(this.dim('Bundle Sizes:'));
+      for (const [bundle, info] of Object.entries(results.bundles)) {
+        const size = this.formatSize(info.size);
+        const gzipSize = info.gzipSize ? ` (${this.formatSize(info.gzipSize)} gzipped)` : '';
+        this.log(`  ${bundle}: ${size}${gzipSize}`);
+      }
+    }
+
+    this.log('');
+  }
+
+  /**
+   * Report regression comparison
+   */
+  reportRegression(results: RegressionResult[]): void {
+    this.log('');
+    this.log(this.header('Regression Analysis'));
+    this.log(this.separator());
+
+    for (const result of results) {
+      const statusIcon =
+        result.status === 'improved'
+          ? this.green('↑')
+          : result.status === 'regressed'
+            ? this.red('↓')
+            : this.gray('→');
+
+      this.log(`${statusIcon} ${this.bright(result.language.toUpperCase())}`);
+
+      // Parse rate delta
+      if (Math.abs(result.parseRateDelta) > 0.01) {
+        const delta =
+          result.parseRateDelta > 0
+            ? this.green(`+${result.parseRateDelta.toFixed(1)}%`)
+            : this.red(`${result.parseRateDelta.toFixed(1)}%`);
+        this.log(`  Parse Rate: ${delta}`);
+      }
+
+      // Confidence delta
+      if (Math.abs(result.avgConfidenceDelta) > 0.01) {
+        const delta =
+          result.avgConfidenceDelta > 0
+            ? this.green(`+${result.avgConfidenceDelta.toFixed(2)}`)
+            : this.red(`${result.avgConfidenceDelta.toFixed(2)}`);
+        this.log(`  Avg Confidence: ${delta}`);
+      }
+
+      // New failures
+      if (result.newFailures.length > 0) {
+        this.log(`  ${this.red('New Failures:')} ${result.newFailures.length}`);
+        if (this.verbose) {
+          result.newFailures.slice(0, 5).forEach(id => {
+            this.log(`    ${this.dim('- ' + id)}`);
+          });
+        }
+      }
+
+      // New successes
+      if (result.newSuccesses.length > 0) {
+        this.log(`  ${this.green('New Successes:')} ${result.newSuccesses.length}`);
+      }
+
+      this.log('');
+    }
+  }
+
+  /**
+   * Report error
+   */
+  reportError(error: Error): void {
+    this.log('');
+    this.log(this.red('✗ Error: ') + error.message);
+    if (this.verbose && error.stack) {
+      this.log(this.dim(error.stack));
+    }
+    this.log('');
+  }
+
+  // =============================================================================
+  // Formatting Helpers
+  // =============================================================================
+
+  private log(message: string): void {
+    console.log(message);
+  }
+
+  private separator(char: string = '━', length: number = 60): string {
+    return this.dim(char.repeat(length));
+  }
+
+  private header(text: string): string {
+    return this.bright(text);
+  }
+
+  private formatSize(bytes: number): string {
+    if (bytes < 1024) return `${bytes} B`;
+    if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  }
+
+  // =============================================================================
+  // Color Helpers
+  // =============================================================================
+
+  private color(text: string, color: string): string {
+    return this.useColors ? `${color}${text}${colors.reset}` : text;
+  }
+
+  private bright(text: string): string {
+    return this.color(text, colors.bright);
+  }
+
+  private dim(text: string): string {
+    return this.color(text, colors.dim);
+  }
+
+  private green(text: string): string {
+    return this.color(text, colors.green);
+  }
+
+  private yellow(text: string): string {
+    return this.color(text, colors.yellow);
+  }
+
+  private red(text: string): string {
+    return this.color(text, colors.red);
+  }
+
+  private cyan(text: string): string {
+    return this.color(text, colors.cyan);
+  }
+
+  private gray(text: string): string {
+    return this.color(text, colors.gray);
+  }
+}
+
+/**
+ * Create a default console reporter
+ */
+export function createConsoleReporter(verbose: boolean = false): ConsoleReporter {
+  return new ConsoleReporter({ verbose, useColors: true });
+}

package/src/multilingual/reporters/json-reporter.ts

@@ -0,0 +1,99 @@
+/**
+ * JSON Reporter - Structured JSON output for programmatic consumption
+ */
+
+import { writeFileSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import type { Reporter, TestConfig, LanguageCode, LanguageResults, TestResults } from '../types';
+
+/**
+ * JSON Reporter
+ *
+ * Outputs structured JSON results to a file.
+ */
+export class JSONReporter implements Reporter {
+  private outputPath: string;
+  private results: TestResults | null = null;
+
+  constructor(outputPath: string = './test-results/results.json') {
+    this.outputPath = outputPath;
+  }
+
+  /**
+   * Report test start (no-op for JSON reporter)
+   */
+  reportStart(_config: TestConfig): void {
+    // No output at start
+  }
+
+  /**
+   * Report language start (no-op for JSON reporter)
+   */
+  reportLanguageStart(_language: LanguageCode, _bundle: string): void {
+    // No output during execution
+  }
+
+  /**
+   * Report language complete (no-op for JSON reporter)
+   */
+  reportLanguageComplete(_results: LanguageResults): void {
+    // No output during execution
+  }
+
+  /**
+   * Report complete test run - write JSON file
+   */
+  reportComplete(results: TestResults): void {
+    this.results = results;
+    this.writeResults();
+  }
+
+  /**
+   * Report error
+   */
+  reportError(error: Error): void {
+    const errorResult = {
+      error: {
+        message: error.message,
+        stack: error.stack,
+        timestamp: new Date().toISOString(),
+      },
+    };
+
+    // Ensure output directory exists
+    const dir = dirname(this.outputPath);
+    mkdirSync(dir, { recursive: true });
+
+    writeFileSync(this.outputPath, JSON.stringify(errorResult, null, 2));
+  }
+
+  /**
+   * Write results to file
+   */
+  private writeResults(): void {
+    if (!this.results) {
+      throw new Error('No results to write');
+    }
+
+    // Ensure output directory exists
+    const dir = dirname(this.outputPath);
+    mkdirSync(dir, { recursive: true });
+
+    const json = JSON.stringify(this.results, null, 2);
+    writeFileSync(this.outputPath, json);
+  }
+
+  /**
+   * Get results
+   */
+  getResults(): TestResults | null {
+    return this.results;
+  }
+}
+
+/**
+ * Create a JSON reporter
+ */
+export function createJSONReporter(outputPath?: string): JSONReporter {
+  return new JSONReporter(outputPath);
+}