@hyperfixi/testing-framework 2.0.0

package/src/multilingual/README.md

@@ -0,0 +1,295 @@
+# Multilingual Testing Framework
+
+Automated validation system for HyperFixi's multilingual hyperscript support across 13 languages.
+
+## Overview
+
+This framework validates the multilingual system by:
+
+- Loading patterns from the patterns-reference database (689 translations)
+- Building or selecting appropriate language bundles
+- Validating parsing across all languages
+- Tracking bundle sizes and performance
+- Comparing against baselines for regression detection
+
+## Quick Start
+
+```bash
+# Test all languages in quick mode (10 patterns/language)
+npm run test:multilingual
+
+# Test specific language with verbose output
+npm run test:multilingual -- --language ja --verbose
+
+# Test multiple languages in full mode
+npm run test:multilingual -- --languages ja,ko,es --full
+
+# Compare against baseline
+npm run test:multilingual -- --regression
+
+# Save current results as new baseline
+npm run test:multilingual -- --save-baseline
+```
+
+## Architecture
+
+```
+multilingual/
+├── cli.ts                    # Command-line interface
+├── orchestrator.ts           # Main test runner
+├── types.ts                  # TypeScript types
+├── pattern-loader.ts         # Database query layer
+├── bundle-builder.ts         # Bundle selection/generation
+├── validators/
+│   ├── parse-validator.ts    # Parse validation
+│   └── size-validator.ts     # Bundle size validation
+└── reporters/
+    ├── console-reporter.ts   # Minimal console output
+    ├── json-reporter.ts      # Structured JSON results
+    └── regression-reporter.ts # Baseline comparison
+```
+
+## Test Flow
+
+1. **Load Configuration** - Parse CLI arguments
+2. **Load Patterns** - Query patterns-reference database
+3. **Select/Build Bundle** - Find or generate appropriate bundle
+4. **Validate Parsing** - Test each pattern with semantic parser
+5. **Report Results** - Output to console, JSON, and regression reports
+
+## CLI Options
+
+### Language Selection
+
+```bash
+-l, --language <code>        # Test specific language (en, ja, es, etc.)
+--languages <codes>          # Test multiple languages (comma-separated)
+```
+
+### Bundle Options
+
+```bash
+-b, --bundle <name>          # Use specific bundle
+--build                      # Build bundle before testing
+```
+
+### Test Modes
+
+```bash
+-m, --mode <mode>            # 'quick' or 'full' (default: quick)
+--quick                      # Quick mode (10 patterns per language)
+--full                       # Full mode (all patterns)
+--limit <n>                  # Patterns per language in quick mode
+```
+
+### Output Options
+
+```bash
+-v, --verbose                # Enable verbose output
+-r, --regression             # Compare to baseline
+--save-baseline              # Save results as new baseline
+```
+
+### Filtering
+
+```bash
+-c, --confidence <n>         # Minimum confidence threshold (0-1)
+--verified-only              # Only test verified translations
+--categories <cats>          # Filter by categories (comma-separated)
+```
+
+## Examples
+
+### Test Japanese with verbose output
+
+```bash
+npm run test:multilingual -- --language ja --verbose
+```
+
+Expected output:
+
+```
+Multilingual Test Runner v1.0.0
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+Languages: ja (1)
+Mode: quick mode (10 patterns/lang)
+
+  ✓ JA: 53/53 (100%)  ⏱  0.8s  conf: 0.92
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Summary: ✓ PASS  53/53  Duration: 0.9s
+```
+
+### Test all priority languages
+
+```bash
+npm run test:multilingual -- --languages en,ja,ko,es --full
+```
+
+### Run regression test
+
+```bash
+npm run test:multilingual -- --regression
+```
+
+Expected regression output:
+
+```
+Regression Analysis
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+→ EN
+  Parse Rate: +2.1%
+  New Successes: 2
+
+↑ JA
+  Parse Rate: +5.3%
+  Avg Confidence: +0.04
+  New Successes: 3
+```
+
+### Save new baseline
+
+```bash
+npm run test:multilingual -- --full --save-baseline
+```
+
+## Programmatic Usage
+
+```typescript
+import { runMultilingualTests } from '@hyperfixi/testing-framework/multilingual';
+
+const results = await runMultilingualTests({
+  languages: ['ja', 'ko'],
+  mode: 'full',
+  regression: true,
+});
+
+console.log(`Tested ${results.summary.totalPatterns} patterns`);
+console.log(
+  `Success rate: ${((results.summary.totalSuccess / results.summary.totalPatterns) * 100).toFixed(1)}%`
+);
+```
+
+## Output Files
+
+### JSON Results
+
+Location: `./test-results/results.json`
+
+Structure:
+
+```json
+{
+  "timestamp": "2026-01-17T10:30:00Z",
+  "commit": "614da020",
+  "languageResults": [
+    {
+      "language": "ja",
+      "parseSuccess": 53,
+      "parseFailure": 0,
+      "parseRate": 1.0,
+      "avgConfidence": 0.92
+    }
+  ],
+  "bundles": {
+    "browser-ja": {
+      "size": 20480,
+      "languages": ["ja"]
+    }
+  }
+}
+```
+
+### Baseline
+
+Location: `./test-results/baseline.json`
+
+Structure:
+
+```json
+{
+  "timestamp": "2026-01-17T10:00:00Z",
+  "commit": "614da020",
+  "languages": {
+    "ja": {
+      "parseSuccess": 51,
+      "parseFailure": 2,
+      "parseRate": 0.96,
+      "avgConfidence": 0.88
+    }
+  }
+}
+```
+
+## Supported Languages
+
+| Code | Language   | Word Order | Status             |
+| ---- | ---------- | ---------- | ------------------ |
+| en   | English    | SVO        | ✅ High coverage   |
+| ja   | Japanese   | SOV        | ✅ High coverage   |
+| ko   | Korean     | SOV        | ✅ High coverage   |
+| es   | Spanish    | SVO        | ✅ High coverage   |
+| zh   | Chinese    | SVO        | ✅ High coverage   |
+| ar   | Arabic     | VSO        | ✅ High coverage   |
+| pt   | Portuguese | SVO        | ✅ Medium coverage |
+| tr   | Turkish    | SOV        | ✅ Medium coverage |
+| de   | German     | V2         | ✅ Medium coverage |
+| fr   | French     | SVO        | ✅ Medium coverage |
+| id   | Indonesian | SVO        | ✅ Medium coverage |
+| qu   | Quechua    | SOV        | ⚠️ Low coverage    |
+| sw   | Swahili    | SVO        | ⚠️ Low coverage    |
+
+## Integration with CI
+
+Add to `.github/workflows/test.yml`:
+
+```yaml
+- name: Run Multilingual Tests
+  run: |
+    npm run test:multilingual -- --full --regression
+```
+
+## Troubleshooting
+
+### Bundle not found
+
+If you see "Bundle not found", build it first:
+
+```bash
+npm run test:multilingual -- --build --language ja
+```
+
+### Patterns database not populated
+
+Ensure the patterns database is populated:
+
+```bash
+cd packages/patterns-reference
+npm run populate
+```
+
+### Low confidence scores
+
+Enable verbose mode to see which patterns are failing:
+
+```bash
+npm run test:multilingual -- --verbose --language ja
+```
+
+## Future Enhancements
+
+- [ ] Browser execution tests (Playwright)
+- [ ] Performance benchmarks
+- [ ] Visual regression for multilingual examples
+- [ ] Auto-fix suggestions for common translation errors
+- [ ] Coverage heatmap visualization
+
+## Contributing
+
+When adding a new language:
+
+1. Add patterns to patterns-reference database
+2. Generate translations with `npm run populate`
+3. Run tests: `npm run test:multilingual -- --language <code>`
+4. Save baseline: `npm run test:multilingual -- --language <code> --save-baseline`

package/src/multilingual/bundle-builder.ts

@@ -0,0 +1,345 @@
+/**
+ * Bundle Builder - Manages language bundle generation and selection
+ */
+
+import { existsSync, statSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
+import { fileURLToPath } from 'node:url';
+import type { LanguageCode, BundleInfo, BundleBuildOptions } from './types';
+
+const execAsync = promisify(exec);
+
+// ESM __dirname equivalent
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Predefined bundle mappings
+const PREDEFINED_BUNDLES = new Map<string, LanguageCode[]>([
+  // All 23 languages
+  [
+    'browser',
+    [
+      'en',
+      'es',
+      'ja',
+      'ar',
+      'ko',
+      'zh',
+      'tr',
+      'pt',
+      'fr',
+      'de',
+      'id',
+      'qu',
+      'sw',
+      'bn',
+      'hi',
+      'it',
+      'ms',
+      'pl',
+      'ru',
+      'th',
+      'tl',
+      'uk',
+      'vi',
+    ],
+  ],
+  ['browser-priority', ['en', 'es', 'pt', 'fr', 'de', 'ja', 'zh', 'ko', 'ar', 'tr', 'id']], // 11 priority
+  ['browser-western', ['en', 'es', 'pt', 'fr', 'de']], // Western languages
+  ['browser-east-asian', ['ja', 'zh', 'ko']], // East Asian languages
+  // Individual language bundles
+  ['browser-ar', ['ar']],
+  ['browser-bn', ['bn']],
+  ['browser-de', ['de']],
+  ['browser-en', ['en']],
+  ['browser-es', ['es']],
+  ['browser-fr', ['fr']],
+  ['browser-hi', ['hi']],
+  ['browser-id', ['id']],
+  ['browser-it', ['it']],
+  ['browser-ja', ['ja']],
+  ['browser-ko', ['ko']],
+  ['browser-ms', ['ms']],
+  ['browser-pl', ['pl']],
+  ['browser-pt', ['pt']],
+  ['browser-qu', ['qu']],
+  ['browser-ru', ['ru']],
+  ['browser-sw', ['sw']],
+  ['browser-th', ['th']],
+  ['browser-tl', ['tl']],
+  ['browser-tr', ['tr']],
+  ['browser-uk', ['uk']],
+  ['browser-vi', ['vi']],
+  ['browser-zh', ['zh']],
+  // Utility bundles
+  ['browser-core', []], // Core bundle (language-agnostic)
+  ['browser-lazy', []], // Lazy-loading bundle
+  ['browser-en-tr', ['en', 'tr']], // English + Turkish
+  ['browser-es-en', ['es', 'en']], // Spanish + English
+]);
+
+/**
+ * Get bundle path in semantic package
+ */
+function getBundlePath(bundleName: string): string {
+  // Resolve from this file's location: testing-framework/src/multilingual
+  // __dirname = /path/to/hyperfixi/packages/testing-framework/src/multilingual
+  // Go up 3 levels to project root, then into packages/semantic
+  const projectRoot = join(__dirname, '../../..');
+  const semanticRoot = join(projectRoot, 'semantic');
+  // Bundle naming: browser-{group}.{group}.global.js
+  const group = bundleName.replace('browser-', '');
+  return join(semanticRoot, 'dist', `${bundleName}.${group}.global.js`);
+}
+
+/**
+ * Find existing bundle for languages
+ */
+export async function findBundleForLanguages(
+  languages: LanguageCode[]
+): Promise<BundleInfo | null> {
+  const langSet = new Set(languages);
+
+  // Find the smallest predefined bundle that contains all required languages
+  let bestBundle: { name: string; langs: LanguageCode[] } | null = null;
+  let minSize = Infinity;
+
+  for (const [bundleName, bundleLangs] of PREDEFINED_BUNDLES.entries()) {
+    const bundleSet = new Set(bundleLangs);
+
+    // Check if bundle contains all required languages
+    const containsAll = languages.every(lang => bundleSet.has(lang));
+    if (!containsAll) continue;
+
+    // Prefer smaller bundles
+    if (bundleLangs.length < minSize) {
+      minSize = bundleLangs.length;
+      bestBundle = { name: bundleName, langs: bundleLangs };
+    }
+  }
+
+  if (!bestBundle) {
+    return null;
+  }
+
+  const bundlePath = getBundlePath(bestBundle.name);
+  const exists = existsSync(bundlePath);
+
+  return {
+    name: bestBundle.name,
+    path: bundlePath,
+    languages: bestBundle.langs,
+    size: exists ? statSync(bundlePath).size : 0,
+    exists,
+  };
+}
+
+/**
+ * Get bundle info by name
+ */
+export async function getBundleInfo(bundleName: string): Promise<BundleInfo | null> {
+  const languages = PREDEFINED_BUNDLES.get(bundleName);
+  if (!languages) {
+    return null;
+  }
+
+  const bundlePath = getBundlePath(bundleName);
+  const exists = existsSync(bundlePath);
+
+  return {
+    name: bundleName,
+    path: bundlePath,
+    languages,
+    size: exists ? statSync(bundlePath).size : 0,
+    exists,
+  };
+}
+
+/**
+ * Build bundle for languages
+ */
+export async function buildBundle(options: BundleBuildOptions): Promise<BundleInfo> {
+  const { languages, groupName } = options;
+
+  // Generate bundle using the generate-bundle.mjs script
+  const semanticRoot = join(process.cwd(), 'packages/semantic');
+  const scriptPath = join(semanticRoot, 'scripts/generate-bundle.mjs');
+
+  let bundleName: string;
+
+  try {
+    if (groupName) {
+      // Use predefined group
+      const cmd = `node ${scriptPath} --group ${groupName} ${options.updateConfig ? '--auto' : ''}`;
+      await execAsync(cmd, { cwd: semanticRoot });
+      bundleName = `browser-${groupName}`;
+    } else {
+      // Use specific languages
+      const langCodes = languages.join(' ');
+      const cmd = `node ${scriptPath} ${langCodes} ${options.updateConfig ? '--auto' : ''}`;
+      await execAsync(cmd, { cwd: semanticRoot });
+      bundleName = `browser-${languages.join('-')}`;
+    }
+
+    // Build the bundle
+    await execAsync('npm run build', { cwd: semanticRoot });
+
+    const bundlePath = options.outputPath || getBundlePath(bundleName);
+    const exists = existsSync(bundlePath);
+
+    return {
+      name: bundleName,
+      path: bundlePath,
+      languages,
+      size: exists ? statSync(bundlePath).size : 0,
+      exists,
+    };
+  } catch (error) {
+    throw new Error(
+      `Failed to build bundle: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
+/**
+ * Select or build bundle for testing
+ */
+export async function selectBundle(
+  languages: LanguageCode[],
+  build: boolean = false
+): Promise<BundleInfo> {
+  // First, try to find existing bundle
+  let bundle = await findBundleForLanguages(languages);
+
+  // If bundle exists and we don't need to build, use it
+  if (bundle && bundle.exists && !build) {
+    return bundle;
+  }
+
+  // If build is requested or no bundle exists, build it
+  if (build || !bundle || !bundle.exists) {
+    // Determine if we can use a predefined group
+    const groupName = findGroupForLanguages(languages);
+
+    bundle = await buildBundle({
+      languages,
+      groupName: groupName !== null ? groupName : undefined,
+      outputPath: undefined,
+      updateConfig: false, // Don't modify configs during testing
+    });
+  }
+
+  if (!bundle.exists) {
+    throw new Error(`Bundle not found: ${bundle.path}`);
+  }
+
+  return bundle;
+}
+
+/**
+ * Find predefined group name for languages
+ */
+function findGroupForLanguages(languages: LanguageCode[]): string | null {
+  const langSet = new Set(languages);
+
+  for (const [bundleName, bundleLangs] of PREDEFINED_BUNDLES.entries()) {
+    const bundleSet = new Set(bundleLangs);
+
+    // Exact match
+    if (bundleLangs.length === languages.length && languages.every(lang => bundleSet.has(lang))) {
+      return bundleName.replace('browser-', '');
+    }
+  }
+
+  return null;
+}
+
+/**
+ * List all available bundles
+ */
+export async function listAvailableBundles(): Promise<BundleInfo[]> {
+  const bundles: BundleInfo[] = [];
+
+  for (const [bundleName, languages] of PREDEFINED_BUNDLES.entries()) {
+    const bundlePath = getBundlePath(bundleName);
+    const exists = existsSync(bundlePath);
+
+    bundles.push({
+      name: bundleName,
+      path: bundlePath,
+      languages,
+      size: exists ? statSync(bundlePath).size : 0,
+      exists,
+    });
+  }
+
+  return bundles;
+}
+
+/**
+ * Check if bundle exists
+ */
+export async function bundleExists(bundleName: string): Promise<boolean> {
+  const bundlePath = getBundlePath(bundleName);
+  return existsSync(bundlePath);
+}
+
+/**
+ * Get bundle size in bytes
+ */
+export async function getBundleSize(bundleName: string): Promise<number> {
+  const bundlePath = getBundlePath(bundleName);
+  if (!existsSync(bundlePath)) {
+    return 0;
+  }
+  return statSync(bundlePath).size;
+}
+
+/**
+ * Estimate bundle size for languages (without building)
+ */
+export function estimateBundleSize(languages: LanguageCode[]): {
+  estimatedSize: number;
+  estimatedGzipSize: number;
+} {
+  const SIZES: Record<string, number> = {
+    en: 25,
+    es: 22,
+    ja: 28,
+    ar: 24,
+    ko: 22,
+    zh: 20,
+    tr: 20,
+    pt: 22,
+    fr: 22,
+    de: 24,
+    id: 18,
+    qu: 16,
+    sw: 18,
+    bn: 20,
+    hi: 20,
+    it: 22,
+    ms: 18,
+    pl: 20,
+    ru: 22,
+    th: 20,
+    tl: 18,
+    uk: 22,
+    vi: 20,
+    base: 45,
+  };
+
+  const GZIP_RATIO = 0.35;
+
+  let totalSize = SIZES['base'] || 45; // Base infrastructure
+  for (const lang of languages) {
+    totalSize += SIZES[lang] || 20; // Default to 20 KB if unknown
+  }
+
+  return {
+    estimatedSize: totalSize * 1024, // Convert to bytes
+    estimatedGzipSize: Math.round(totalSize * 1024 * GZIP_RATIO),
+  };
+}