@fragments-sdk/cli 0.2.2

package/src/test/runner.ts

@@ -0,0 +1,598 @@
+/**
+ * Test runner - executes play functions in browser via Playwright
+ *
+ * Playwright is an optional dependency. If not installed, the test command
+ * will prompt the user to install it.
+ */
+
+import type {
+  TestCase,
+  TestResult,
+  TestSuite,
+  TestRunResult,
+  TestError,
+  StepResult,
+  A11yResult,
+  A11yViolation,
+  RunnerOptions,
+  TestReporter,
+} from './types.js';
+import { groupTestsByComponent } from './discovery.js';
+
+// Dynamic playwright types (since it's optional)
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Browser = any;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type BrowserContext = any;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Page = any;
+
+/**
+ * Browser pool for parallel test execution
+ */
+interface BrowserPool {
+  browser: Browser;
+  contexts: BrowserContextEntry[];
+}
+
+interface BrowserContextEntry {
+  id: number;
+  context: BrowserContext;
+  page: Page;
+  busy: boolean;
+}
+
+/**
+ * Run tests with the given configuration
+ */
+export async function runTests(
+  testCases: TestCase[],
+  options: RunnerOptions,
+  reporters: TestReporter[] = []
+): Promise<TestRunResult> {
+  const startTime = new Date();
+  const results: TestResult[] = [];
+
+  // Notify reporters of run start
+  for (const reporter of reporters) {
+    reporter.onRunStart?.(testCases.length);
+  }
+
+  // Import Playwright dynamically (optional dependency)
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let playwright: any;
+  try {
+    playwright = await import('playwright');
+  } catch {
+    throw new Error(
+      'Playwright is required for running tests. Install it with: npm install -D playwright'
+    );
+  }
+
+  // Launch browser
+  const browserType = playwright[options.browser];
+  const browser = await browserType.launch({
+    headless: options.headless,
+  });
+
+  // Create browser pool for parallel execution
+  const pool = await createBrowserPool(browser, options.parallel);
+
+  try {
+    // Determine server URL
+    const serverUrl = options.serverUrl || `http://localhost:${options.port}`;
+
+    // Run tests with parallelism
+    if (options.parallel > 1) {
+      await runTestsParallel(
+        testCases,
+        pool,
+        serverUrl,
+        options,
+        reporters,
+        results
+      );
+    } else {
+      await runTestsSequential(
+        testCases,
+        pool.contexts[0],
+        serverUrl,
+        options,
+        reporters,
+        results
+      );
+    }
+  } finally {
+    // Cleanup browser pool
+    await cleanupBrowserPool(pool);
+  }
+
+  const endTime = new Date();
+
+  // Aggregate results into suites
+  const suites = aggregateResults(results);
+
+  const runResult: TestRunResult = {
+    suites,
+    totalTests: results.length,
+    totalPassed: results.filter((r) => r.status === 'passed').length,
+    totalFailed: results.filter((r) => r.status === 'failed').length,
+    totalSkipped: results.filter((r) => r.status === 'skipped').length,
+    totalDuration: endTime.getTime() - startTime.getTime(),
+    startTime,
+    endTime,
+    totalA11yViolations: options.a11y
+      ? results.reduce((sum, r) => sum + (r.accessibility?.violations.length || 0), 0)
+      : undefined,
+  };
+
+  // Notify reporters of run completion
+  for (const reporter of reporters) {
+    await reporter.onRunComplete(runResult);
+  }
+
+  return runResult;
+}
+
+/**
+ * Create a browser pool for parallel execution
+ */
+async function createBrowserPool(
+  browser: Browser,
+  parallelism: number
+): Promise<BrowserPool> {
+  const contexts: BrowserContextEntry[] = [];
+
+  for (let i = 0; i < parallelism; i++) {
+    const context = await browser.newContext({
+      viewport: { width: 1280, height: 720 },
+    });
+    const page = await context.newPage();
+    contexts.push({ id: i, context, page, busy: false });
+  }
+
+  return { browser, contexts };
+}
+
+/**
+ * Cleanup browser pool
+ */
+async function cleanupBrowserPool(pool: BrowserPool): Promise<void> {
+  for (const entry of pool.contexts) {
+    await entry.page.close();
+    await entry.context.close();
+  }
+  await pool.browser.close();
+}
+
+/**
+ * Run tests sequentially
+ */
+async function runTestsSequential(
+  testCases: TestCase[],
+  ctx: BrowserContextEntry,
+  serverUrl: string,
+  options: RunnerOptions,
+  reporters: TestReporter[],
+  results: TestResult[]
+): Promise<void> {
+  for (const testCase of testCases) {
+    // Notify reporters of test start
+    for (const reporter of reporters) {
+      reporter.onTestStart?.(testCase);
+    }
+
+    const result = await runSingleTest(testCase, ctx, serverUrl, options);
+    results.push(result);
+
+    // Notify reporters of test completion
+    for (const reporter of reporters) {
+      reporter.onTestComplete?.(result);
+    }
+
+    // Bail on first failure if configured
+    if (options.bail && result.status === 'failed') {
+      break;
+    }
+  }
+}
+
+/**
+ * Run tests in parallel
+ */
+async function runTestsParallel(
+  testCases: TestCase[],
+  pool: BrowserPool,
+  serverUrl: string,
+  options: RunnerOptions,
+  reporters: TestReporter[],
+  results: TestResult[]
+): Promise<void> {
+  const queue = [...testCases];
+  const inFlight: Promise<void>[] = [];
+  let bailTriggered = false;
+
+  const acquireContext = (): BrowserContextEntry | null => {
+    const available = pool.contexts.find((c) => !c.busy);
+    if (available) {
+      available.busy = true;
+      return available;
+    }
+    return null;
+  };
+
+  const releaseContext = (ctx: BrowserContextEntry): void => {
+    ctx.busy = false;
+  };
+
+  const runNext = async (): Promise<void> => {
+    if (bailTriggered || queue.length === 0) return;
+
+    const ctx = acquireContext();
+    if (!ctx) return;
+
+    const testCase = queue.shift()!;
+
+    // Notify reporters of test start
+    for (const reporter of reporters) {
+      reporter.onTestStart?.(testCase);
+    }
+
+    try {
+      const result = await runSingleTest(testCase, ctx, serverUrl, options);
+      results.push(result);
+
+      // Notify reporters of test completion
+      for (const reporter of reporters) {
+        reporter.onTestComplete?.(result);
+      }
+
+      // Check bail condition
+      if (options.bail && result.status === 'failed') {
+        bailTriggered = true;
+      }
+    } finally {
+      releaseContext(ctx);
+      // Start next test if not bailing
+      if (!bailTriggered && queue.length > 0) {
+        const nextPromise = runNext();
+        inFlight.push(nextPromise);
+      }
+    }
+  };
+
+  // Start initial batch
+  const initialBatch = Math.min(pool.contexts.length, queue.length);
+  for (let i = 0; i < initialBatch; i++) {
+    inFlight.push(runNext());
+  }
+
+  // Wait for all to complete
+  await Promise.all(inFlight);
+}
+
+/**
+ * Run a single test case
+ */
+async function runSingleTest(
+  testCase: TestCase,
+  ctx: BrowserContextEntry,
+  serverUrl: string,
+  options: RunnerOptions
+): Promise<TestResult> {
+  const { page } = ctx;
+  const startTime = performance.now();
+  const steps: StepResult[] = [];
+  let error: TestError | undefined;
+  let status: TestResult['status'] = 'passed';
+  let retryAttempt = 0;
+
+  // Retry logic
+  const maxAttempts = options.retries + 1;
+
+  while (retryAttempt < maxAttempts) {
+    try {
+      // Navigate to the variant
+      const url = buildVariantUrl(serverUrl, testCase);
+      await page.goto(url, { waitUntil: 'networkidle' });
+
+      // Wait for preview container
+      await page.waitForSelector('[data-preview-container="true"]', {
+        timeout: options.timeout,
+      });
+
+      // Execute play function in browser context
+      const playResult = await executePlayFunction(page, testCase, options.timeout);
+
+      if (playResult.error) {
+        error = playResult.error;
+        status = 'failed';
+        steps.push(...playResult.steps);
+
+        // Retry if not last attempt
+        if (retryAttempt < maxAttempts - 1) {
+          retryAttempt++;
+          continue;
+        }
+      } else {
+        status = 'passed';
+        steps.push(...playResult.steps);
+      }
+
+      break; // Success, no retry needed
+    } catch (e) {
+      error = {
+        message: e instanceof Error ? e.message : String(e),
+        stack: e instanceof Error ? e.stack : undefined,
+      };
+      status = 'failed';
+
+      if (retryAttempt < maxAttempts - 1) {
+        retryAttempt++;
+        continue;
+      }
+      break;
+    }
+  }
+
+  const duration = performance.now() - startTime;
+
+  // Run accessibility checks if enabled
+  let accessibility: A11yResult | undefined;
+  if (options.a11y && status === 'passed') {
+    accessibility = await runA11yChecks(page);
+  }
+
+  // Capture screenshot for visual regression if enabled
+  let screenshotPath: string | undefined;
+  if (options.visual) {
+    screenshotPath = await captureScreenshot(page, testCase, options.outputDir);
+  }
+
+  return {
+    testCase,
+    status,
+    duration,
+    steps,
+    error,
+    accessibility,
+    screenshotPath,
+    retryAttempt: retryAttempt > 0 ? retryAttempt : undefined,
+  };
+}
+
+/**
+ * Build URL for a variant
+ */
+function buildVariantUrl(serverUrl: string, testCase: TestCase): string {
+  const params = new URLSearchParams({
+    component: testCase.component,
+    variant: testCase.variant,
+    isolated: 'true',
+  });
+  return `${serverUrl}?${params.toString()}`;
+}
+
+/**
+ * Execute play function in browser context
+ */
+async function executePlayFunction(
+  page: Page,
+  testCase: TestCase,
+  timeout: number
+): Promise<{ steps: StepResult[]; error?: TestError }> {
+  const result = await page.evaluate(
+    async ({ component, variant, timeout }: { component: string; variant: string; timeout: number }) => {
+      // Find the segment definition in the global registry
+      const registry = (window as unknown as { __SEGMENTS_REGISTRY__?: Map<string, unknown> })
+        .__SEGMENTS_REGISTRY__;
+
+      if (!registry) {
+        return {
+          steps: [],
+          error: { message: 'Segments registry not found. Make sure the viewer is loaded.' },
+        };
+      }
+
+      const segment = registry.get(component) as {
+        variants?: Array<{
+          name: string;
+          play?: (ctx: {
+            canvasElement: HTMLElement;
+            args: Record<string, unknown>;
+            step: (name: string, fn: () => Promise<void>) => Promise<void>;
+          }) => Promise<void>;
+        }>;
+      } | undefined;
+
+      if (!segment) {
+        return {
+          steps: [],
+          error: { message: `Component "${component}" not found in registry` },
+        };
+      }
+
+      const variantDef = segment.variants?.find((v) => v.name === variant);
+      if (!variantDef || !variantDef.play) {
+        return {
+          steps: [],
+          error: { message: `Variant "${variant}" not found or has no play function` },
+        };
+      }
+
+      // Find the preview container
+      const canvasElement = document.querySelector('[data-preview-container="true"]') as HTMLElement;
+      if (!canvasElement) {
+        return {
+          steps: [],
+          error: { message: 'Preview container not found' },
+        };
+      }
+
+      // Track steps
+      const steps: Array<{
+        name: string;
+        status: 'passed' | 'failed';
+        duration: number;
+        error?: { message: string };
+      }> = [];
+
+      const step = async (name: string, fn: () => Promise<void>): Promise<void> => {
+        const stepStart = performance.now();
+        try {
+          await fn();
+          steps.push({
+            name,
+            status: 'passed',
+            duration: performance.now() - stepStart,
+          });
+        } catch (e) {
+          steps.push({
+            name,
+            status: 'failed',
+            duration: performance.now() - stepStart,
+            error: { message: e instanceof Error ? e.message : String(e) },
+          });
+          throw e;
+        }
+      };
+
+      // Execute with timeout
+      const timeoutPromise = new Promise<never>((_, reject) => {
+        setTimeout(() => reject(new Error(`Test timed out after ${timeout}ms`)), timeout);
+      });
+
+      try {
+        await Promise.race([
+          variantDef.play({
+            canvasElement,
+            args: {},
+            step,
+          }),
+          timeoutPromise,
+        ]);
+
+        // If no explicit steps, add one for the whole play function
+        if (steps.length === 0) {
+          steps.push({
+            name: 'Play function',
+            status: 'passed',
+            duration: 0,
+          });
+        }
+
+        return { steps, error: undefined };
+      } catch (e) {
+        // If no steps failed explicitly, add one
+        if (!steps.some((s) => s.status === 'failed')) {
+          steps.push({
+            name: 'Play function',
+            status: 'failed',
+            duration: 0,
+            error: { message: e instanceof Error ? e.message : String(e) },
+          });
+        }
+
+        return {
+          steps,
+          error: { message: e instanceof Error ? e.message : String(e) },
+        };
+      }
+    },
+    { component: testCase.component, variant: testCase.variant, timeout }
+  );
+
+  return result as { steps: StepResult[]; error?: TestError };
+}
+
+/**
+ * Run accessibility checks using axe-core
+ */
+async function runA11yChecks(page: Page): Promise<A11yResult> {
+  try {
+    // Inject axe-core if not already present
+    await page.evaluate(() => {
+      if ((window as unknown as { axe?: unknown }).axe) return;
+
+      return new Promise<void>((resolve, reject) => {
+        const script = document.createElement('script');
+        script.src = 'https://cdnjs.cloudflare.com/ajax/libs/axe-core/4.8.2/axe.min.js';
+        script.onload = () => resolve();
+        script.onerror = () => reject(new Error('Failed to load axe-core'));
+        document.head.appendChild(script);
+      });
+    });
+
+    // Run axe-core analysis
+    const results = await page.evaluate(async () => {
+      const axe = (window as unknown as { axe: { run: (el: HTMLElement) => Promise<{ violations: unknown[]; passes: unknown[] }> } }).axe;
+      const container = document.querySelector('[data-preview-container="true"]') as HTMLElement;
+      if (!container) {
+        return { violations: [], passes: 0 };
+      }
+      const axeResults = await axe.run(container);
+      return {
+        violations: axeResults.violations,
+        passes: axeResults.passes.length,
+      };
+    });
+
+    return {
+      violations: results.violations as A11yViolation[],
+      passes: results.passes,
+    };
+  } catch {
+    return { violations: [], passes: 0 };
+  }
+}
+
+/**
+ * Capture screenshot for visual regression
+ */
+async function captureScreenshot(
+  page: Page,
+  testCase: TestCase,
+  outputDir: string
+): Promise<string> {
+  const { mkdir, writeFile } = await import('node:fs/promises');
+  const { join } = await import('node:path');
+
+  const screenshotsDir = join(outputDir, 'screenshots');
+  await mkdir(screenshotsDir, { recursive: true });
+
+  const filename = `${testCase.component}--${testCase.variant}.png`
+    .replace(/\s+/g, '-')
+    .toLowerCase();
+  const filepath = join(screenshotsDir, filename);
+
+  const container = await page.$('[data-preview-container="true"]');
+  if (container) {
+    const screenshot = await container.screenshot();
+    await writeFile(filepath, screenshot);
+  }
+
+  return filepath;
+}
+
+/**
+ * Aggregate test results into suites by component
+ */
+function aggregateResults(results: TestResult[]): TestSuite[] {
+  const grouped = groupTestsByComponent(results.map((r) => r.testCase));
+  const suites: TestSuite[] = [];
+
+  for (const [componentName, testCases] of grouped) {
+    const componentResults = results.filter((r) => r.testCase.component === componentName);
+
+    suites.push({
+      name: componentName,
+      tests: componentResults,
+      duration: componentResults.reduce((sum, r) => sum + r.duration, 0),
+      passed: componentResults.filter((r) => r.status === 'passed').length,
+      failed: componentResults.filter((r) => r.status === 'failed').length,
+      skipped: componentResults.filter((r) => r.status === 'skipped').length,
+    });
+  }
+
+  return suites;
+}