creevey 0.10.0-beta.43 → 0.10.0-beta.44

package/src/server/master/testsManager.ts

@@ -0,0 +1,315 @@
+import path from 'path';
+import { mkdirSync, writeFileSync } from 'fs';
+import EventEmitter from 'events';
+import {
+  ServerTest,
+  TestMeta,
+  TestResult,
+  TestStatus,
+  CreeveyUpdate,
+  ApprovePayload,
+  isDefined,
+  isFunction,
+  CreeveyStatus,
+} from '../../types.js';
+import { tryToLoadTestsData } from '../utils.js';
+import { copyFile, mkdir, writeFile } from 'fs/promises';
+
+/**
+ * TestsManager is responsible for all operations related to test data management
+ * including loading, saving, merging, and updating test data.
+ * It extends EventEmitter to emit update events that can be subscribed to.
+ */
+export class TestsManager extends EventEmitter {
+  private tests: Partial<Record<string, ServerTest>> = {};
+  private screenDir: string;
+  private reportDir: string;
+
+  /**
+   * Creates a new TestsManager instance
+   * @param screenDir Directory for storing reference images
+   * @param reportDir Directory for storing reports and screenshots
+   */
+  constructor(screenDir: string, reportDir: string) {
+    super();
+    this.screenDir = screenDir;
+    this.reportDir = reportDir;
+  }
+
+  /**
+   * Get a copy of all tests
+   * @returns all tests
+   */
+  public getTests(): Partial<Record<string, ServerTest>> {
+    return this.tests;
+  }
+
+  /**
+   * Get a test by ID
+   * @param id Test ID
+   * @returns Test data
+   */
+  public getTest(id: string): ServerTest | undefined {
+    return this.tests[id];
+  }
+
+  /**
+   * Get test data in a format suitable for status reporting
+   * @returns Test data in the format needed for status
+   */
+  public getTestsData(): CreeveyStatus['tests'] {
+    const testsData: CreeveyStatus['tests'] = {};
+
+    Object.entries(this.tests).forEach(([id, test]) => {
+      if (!test) return;
+
+      const { story: _, fn: __, ...testData } = test;
+      testsData[id] = testData;
+    });
+
+    return testsData;
+  }
+
+  /**
+   * Load tests from a report file
+   */
+  public loadTestsFromReport(): Partial<Record<string, ServerTest>> {
+    const reportDataPath = path.join(this.reportDir, 'data.js');
+    const testsFromReport = tryToLoadTestsData(reportDataPath) ?? {};
+    return testsFromReport;
+  }
+
+  /**
+   * Merge tests from report with tests from stories
+   */
+  private mergeTests(
+    testsWithReports: CreeveyStatus['tests'],
+    testsFromStories: Partial<Record<string, ServerTest>>,
+  ): Partial<Record<string, ServerTest>> {
+    Object.values(testsFromStories)
+      .filter(isDefined)
+      .forEach((test) => {
+        const testWithReport = testsWithReports[test.id];
+        if (!testWithReport) return;
+        test.retries = testWithReport.retries;
+        if (testWithReport.status === 'success' || testWithReport.status === 'failed') {
+          test.status = testWithReport.status;
+        }
+        test.results = testWithReport.results;
+        test.approved = testWithReport.approved;
+      });
+
+    return testsFromStories;
+  }
+
+  public loadAndMergeTests(testsFromStories: Partial<Record<string, ServerTest>>): Partial<Record<string, ServerTest>> {
+    const testsFromReport = this.loadTestsFromReport();
+
+    return this.mergeTests(testsFromReport, testsFromStories);
+  }
+
+  /**
+   * Update tests with incremental changes
+   * @param testsDiff Tests to update or remove
+   */
+  public updateTests(testsDiff: Partial<Record<string, ServerTest>>): CreeveyUpdate | null {
+    const tests: CreeveyUpdate['tests'] = {};
+    const removedTests: TestMeta[] = [];
+
+    Object.entries(testsDiff).forEach(([id, newTest]) => {
+      if (newTest) {
+        if (this.tests[id]) {
+          this.tests[id] = {
+            ...newTest,
+            retries: this.tests[id].retries,
+            results: this.tests[id].results,
+            approved: this.tests[id].approved,
+          };
+        } else {
+          this.tests[id] = newTest;
+        }
+
+        const { story: _, fn: __, ...restTest } = newTest;
+        tests[id] = { ...restTest, status: 'unknown' };
+      } else if (this.tests[id]) {
+        const { id: testId, browser, testName, storyPath, storyId } = this.tests[id];
+        removedTests.push({ id: testId, browser, testName, storyPath, storyId });
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete this.tests[id];
+      }
+    });
+
+    this.saveTestsToJson();
+
+    const update = { tests, removedTests };
+    this.emit('update', update);
+    return update;
+  }
+
+  /**
+   * Update test result
+   * @param id Test ID
+   * @param status New test status
+   * @param result Optional test result
+   */
+  public updateTestStatus(id: string, status: TestStatus, result?: TestResult): CreeveyUpdate | null {
+    // TODO Handle 'retrying' status
+    const test = this.tests[id];
+    if (!test) return null;
+
+    const { browser, testName, storyPath, storyId } = test;
+    test.status = status === 'retrying' ? 'failed' : status;
+
+    if (!result) {
+      // NOTE: Running status
+      const update = { tests: { [id]: { id, browser, testName, storyPath, status, storyId } } };
+      this.emit('update', update);
+      return update;
+    }
+
+    test.results ??= [];
+    test.results.push(result);
+
+    if (status === 'failed') {
+      test.approved = null;
+    }
+
+    const update = {
+      tests: {
+        [id]: {
+          id,
+          browser,
+          testName,
+          storyPath,
+          status,
+          approved: test.approved,
+          results: [result],
+          storyId,
+        },
+      },
+    };
+
+    this.emit('update', update);
+    return update;
+  }
+
+  /**
+   * Save tests to JSON file
+   * @param reportDir Directory to save the JSON file
+   */
+  public saveTestsToJson(): void {
+    mkdirSync(this.reportDir, { recursive: true });
+    writeFileSync(
+      path.join(this.reportDir, 'tests.json'),
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+      JSON.stringify(this.tests, (_, value) => (isFunction(value) ? value.toString() : value), 2),
+    );
+  }
+
+  /**
+   * Save test data to a module
+   * @param data Test data to include in the module
+   */
+  public async saveTestData(data: CreeveyStatus['tests'] = this.getTestsData()): Promise<void> {
+    const dataModule = `
+(function (root, factory) {
+  if (typeof module === 'object' && module.exports) {
+    module.exports = factory();
+  } else {
+    root.__CREEVEY_DATA__ = factory();
+  }
+}(this, function () { return ${JSON.stringify(data)} }));
+`;
+    await writeFile(path.join(this.reportDir, 'data.js'), dataModule);
+  }
+
+  /**
+   * Copy image for approval
+   * @param test Test data
+   * @param image Image name
+   * @param actual Actual image path
+   */
+  private async copyImage(test: ServerTest, image: string, actual: string): Promise<void> {
+    const { browser, testName, storyPath } = test;
+    const restPath = [...storyPath, testName].filter(isDefined);
+    const testPath = path.join(...restPath, image == browser ? '' : browser);
+    const srcImagePath = path.join(this.reportDir, testPath, actual);
+    const dstImagePath = path.join(this.screenDir, testPath, `${image}.png`);
+    await mkdir(path.join(this.screenDir, testPath), { recursive: true });
+    await copyFile(srcImagePath, dstImagePath);
+  }
+
+  /**
+   * Approve a specific test
+   * @param payload Approval payload with test ID, retry index, and image name
+   */
+  public async approve({ id, retry, image }: ApprovePayload): Promise<CreeveyUpdate | null> {
+    const test = this.tests[id];
+    if (!test?.results) return null;
+    const result = test.results[retry];
+    if (!result.images) return null;
+    const images = result.images[image];
+    if (!images) return null;
+    test.approved ??= {};
+    const { browser, testName, storyPath, storyId } = test;
+
+    await this.copyImage(test, image, images.actual);
+
+    test.approved[image] = retry;
+
+    if (Object.keys(result.images).every((name) => typeof test.approved?.[name] == 'number')) {
+      test.status = 'approved';
+    }
+
+    const update = {
+      tests: {
+        [id]: {
+          id,
+          browser,
+          testName,
+          storyPath,
+          status: test.status,
+          approved: test.approved,
+          storyId,
+        },
+      },
+    };
+
+    this.emit('update', update);
+    return update;
+  }
+
+  /**
+   * Approve all failed tests
+   */
+  public async approveAll(): Promise<CreeveyUpdate> {
+    const updatedTests: NonNullable<CreeveyUpdate['tests']> = {};
+    for (const test of Object.values(this.tests)) {
+      if (!test?.results) continue;
+      const retry = test.results.length - 1;
+      const { images, status } = test.results.at(retry) ?? {};
+      if (!images || status != 'failed') continue;
+      for (const [name, image] of Object.entries(images)) {
+        if (!image) continue;
+        await this.copyImage(test, name, image.actual);
+
+        test.approved ??= {};
+        test.approved[name] = retry;
+        test.status = 'approved';
+
+        updatedTests[test.id] = {
+          id: test.id,
+          browser: test.browser,
+          storyPath: test.storyPath,
+          storyId: test.storyId,
+          status: test.status,
+          approved: { [name]: retry },
+        };
+      }
+    }
+
+    const result = { tests: updatedTests };
+    this.emit('update', result);
+    return result;
+  }
+}

package/src/server/playwright/reporter.ts

@@ -0,0 +1,386 @@
+import type { Reporter, FullConfig, Suite, TestCase, TestResult, TestStep } from '@playwright/test/reporter';
+import path from 'path';
+import fs from 'fs/promises';
+import { TestsManager } from '../master/testsManager.js';
+import { CreeveyApi } from '../master/api.js';
+import { ServerTest, TestMeta, TestStatus, TestResult as CreeveyTestResult } from '../../types.js';
+import { copyStatics } from '../utils.js';
+
+/**
+ * Simple async queue to handle operations in sequence without returning promises
+ * from reporter methods that should be synchronous
+ */
+class AsyncQueue {
+  private queue: Promise<void>;
+
+  constructor() {
+    this.queue = Promise.resolve();
+  }
+
+  /**
+   * Add an async operation to the queue
+   * @param operation Async operation to execute
+   */
+  enqueue(operation: () => Promise<void>): void {
+    this.queue = this.queue.then(operation).catch((error: unknown) => {
+      console.error(`Error in async queue: ${error instanceof Error ? error.message : String(error)}`);
+    });
+  }
+
+  /**
+   * Wait for all operations in the queue to complete
+   */
+  async waitForCompletion(): Promise<void> {
+    await this.queue;
+  }
+}
+
+/**
+ * CreeveyPlaywrightReporter is a Playwright reporter that integrates with Creevey
+ * to provide visual testing capabilities and use Creevey's UI for reviewing and approving screenshots.
+ */
+export class CreeveyPlaywrightReporter implements Reporter {
+  private testsManager: TestsManager;
+  private api: CreeveyApi | null = null;
+  private reportDir: string;
+  private screenDir: string;
+  private port: number;
+  private debug: boolean;
+  private startServer: ((reportDir: string, port: number, uiEnabled: boolean) => (api: CreeveyApi) => void) | null =
+    null;
+  private testIdMap = new Map<string, string>(); // Maps Playwright test IDs to Creevey test IDs
+  private asyncQueue = new AsyncQueue();
+
+  /**
+   * Creates a new instance of the CreeveyPlaywrightReporter
+   * @param options Configuration options for the reporter
+   */
+  constructor(options?: { reportDir?: string; screenDir?: string; port?: number; debug?: boolean }) {
+    this.reportDir = options?.reportDir ?? path.join(process.cwd(), 'report');
+    this.screenDir = options?.screenDir ?? path.join(process.cwd(), 'images');
+    this.port = options?.port ?? 3000;
+    this.debug = options?.debug ?? false;
+
+    // Initialize TestsManager
+    this.testsManager = new TestsManager(this.screenDir, this.reportDir);
+  }
+
+  /**
+   * Called when the test run starts
+   * @param config Playwright configuration
+   * @param suite Test suite information
+   */
+  onBegin(_config: FullConfig, _suite: Suite): void {
+    this.logDebug('CreeveyPlaywrightReporter started');
+
+    // Use the async queue to handle initialization without returning a promise
+    this.asyncQueue.enqueue(async () => {
+      try {
+        // Dynamically import the modules to avoid circular dependencies
+        const { start } = await import('../master/server.js');
+        this.startServer = start;
+
+        // Initialize report directory
+        try {
+          await fs.mkdir(this.reportDir, { recursive: true });
+          await copyStatics(this.reportDir);
+        } catch (error) {
+          this.logError(
+            `Failed to initialize report directory: ${error instanceof Error ? error.message : String(error)}`,
+          );
+        }
+
+        // Start server API
+        try {
+          const resolveApi = this.startServer(this.reportDir, this.port, true);
+
+          // Create and connect the API
+          this.api = new CreeveyApi(this.testsManager);
+          resolveApi(this.api);
+
+          console.log(`Creevey report server started at http://localhost:${this.port}`);
+        } catch (error) {
+          this.logError(`Could not start Creevey server: ${error instanceof Error ? error.message : String(error)}`);
+          console.log('Screenshots will still be captured but UI will not be available');
+        }
+      } catch (error) {
+        this.logError(
+          `Error in Creevey reporter initialization: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    });
+  }
+
+  /**
+   * Called when a test begins
+   * @param test Test case information
+   * @param result Test result (initially empty)
+   */
+  onTestBegin(test: TestCase, _result: TestResult): void {
+    try {
+      // Map test to Creevey test format
+      const creeveyTest = this.mapToCreeveyTest(test);
+      if (!creeveyTest) return;
+
+      // Create a mapping from Playwright test ID to Creevey test ID
+      this.testIdMap.set(test.id, creeveyTest.id);
+
+      // Update test status to running
+      this.testsManager.updateTestStatus(creeveyTest.id, 'running');
+
+      this.logDebug(`Test started: ${test.title} (${creeveyTest.id})`);
+    } catch (error) {
+      this.logError(`Error in onTestBegin: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  /**
+   * Called when a test step begins
+   * @param test Test case information
+   * @param result Test result
+   * @param step Test step information
+   */
+  onStepBegin(test: TestCase, _result: TestResult, step: TestStep): void {
+    if (this.debug) {
+      this.logDebug(`Step started: ${step.title} in test: ${test.title}`);
+    }
+  }
+
+  /**
+   * Called when a test step ends
+   * @param test Test case information
+   * @param result Test result
+   * @param step Test step information
+   */
+  onStepEnd(_test: TestCase, _result: TestResult, step: TestStep): void {
+    try {
+      // If step has attachments, process them
+      if (step.attachments.length > 0) {
+        this.logDebug(`Processing ${step.attachments.length} attachments from step: ${step.title}`);
+
+        // We'll process attachments in onTestEnd for simplicity in this initial implementation
+      }
+    } catch (error) {
+      this.logError(`Error in onStepEnd: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  /**
+   * Called when a test ends
+   * @param test Test case information
+   * @param result Test result
+   */
+  onTestEnd(test: TestCase, result: TestResult): void {
+    const creeveyTestId = this.testIdMap.get(test.id);
+
+    // Use the async queue to handle result processing without returning a promise
+    this.asyncQueue.enqueue(async () => {
+      try {
+        // Process test results and screenshots
+        await this.processTestResult(test, result);
+
+        if (creeveyTestId) {
+          this.logDebug(`Test ended: ${test.title} (${creeveyTestId}) with status: ${result.status}`);
+        }
+      } catch (error) {
+        this.logError(`Error in onTestEnd: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    });
+  }
+
+  /**
+   * Called when the test run ends
+   * @param result The overall test run result
+   */
+  onEnd(result: { status: 'passed' | 'failed' | 'timedout' | 'interrupted' }): void {
+    // Use the async queue to handle final operations without returning a promise
+    this.asyncQueue.enqueue(async () => {
+      try {
+        // Wait for all previous operations to complete
+        await this.asyncQueue.waitForCompletion();
+
+        // Save test data
+        await this.testsManager.saveTestData();
+
+        this.logDebug(`Test run ended with status: ${result.status}`);
+        console.log(`Visual test results available at http://localhost:${this.port}`);
+
+        // No cleanup of server here as it needs to stay running for the user to view results
+      } catch (error) {
+        this.logError(`Error during reporter cleanup: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    });
+  }
+
+  /**
+   * Maps a Playwright test to a Creevey test format
+   * @param test Playwright test case
+   * @returns Creevey test object or null if mapping fails
+   */
+  private mapToCreeveyTest(test: TestCase): ServerTest | null {
+    try {
+      // Try to extract Creevey metadata from annotations
+      let testName = test.title;
+      let browser = 'chromium'; // Default browser
+      let storyPath: string[] = [];
+
+      const creeveyAnnotation = test.annotations.find((a) => a.type === 'creevey');
+      if (creeveyAnnotation?.description) {
+        try {
+          const metadata = JSON.parse(creeveyAnnotation.description) as {
+            testName?: string;
+            browser?: string;
+            storyPath?: string[];
+          };
+          if (metadata.testName) testName = metadata.testName;
+          if (metadata.browser) browser = metadata.browser;
+          if (metadata.storyPath) storyPath = metadata.storyPath;
+        } catch (e) {
+          this.logError(`Failed to parse Creevey metadata: ${e instanceof Error ? e.message : String(e)}`);
+        }
+      }
+
+      // If no explicit storyPath, use the project and file path
+      if (storyPath.length === 0) {
+        const projectName = test.parent.project()?.name;
+        const titlePath = test.titlePath().slice(0, -1); // Exclude the test title itself
+        storyPath = projectName ? [projectName, ...titlePath] : titlePath;
+      }
+
+      // Generate a unique test ID
+      const testId = `${storyPath.join('/')}/${testName}/${browser}`;
+
+      // Create the test metadata
+      const testMeta: TestMeta = {
+        id: testId,
+        storyPath,
+        browser,
+        testName,
+        storyId: storyPath.join('/'),
+      };
+
+      // Create a stub ServerTest object
+      // This is missing the story and fn properties which would be used in a real Creevey test
+      // However, for our reporter purposes, we just need the metadata
+      const serverTest: ServerTest = {
+        ...testMeta,
+        story: {
+          parameters: {},
+          initialArgs: {},
+          argTypes: {},
+          component: '',
+          componentId: '',
+          name: '',
+          tags: [],
+          title: '',
+          kind: '',
+          id: '',
+          story: '',
+        }, // Placeholder
+        fn: async () => {
+          /* Empty function as placeholder */
+        }, // Placeholder
+      };
+
+      return serverTest;
+    } catch (error) {
+      this.logError(`Error mapping test to Creevey format: ${error instanceof Error ? error.message : String(error)}`);
+      return null;
+    }
+  }
+
+  /**
+   * Process a test result and any attachments
+   * @param test Playwright test case
+   * @param result Playwright test result
+   */
+  private async processTestResult(test: TestCase, result: TestResult): Promise<void> {
+    const creeveyTestId = this.testIdMap.get(test.id);
+    if (!creeveyTestId) {
+      this.logError(`No Creevey test ID found for test: ${test.title}`);
+      return;
+    }
+
+    // Determine test status
+    let status: TestStatus;
+    switch (result.status) {
+      case 'passed':
+        status = 'success';
+        break;
+      case 'failed':
+      case 'timedOut':
+        status = 'failed';
+        break;
+      default:
+        status = 'unknown';
+    }
+
+    // Process attachments
+    const images: Record<string, { actual: string }> = {};
+    const attachmentPaths: string[] = [];
+
+    if (result.attachments.length > 0) {
+      await fs.mkdir(path.join(this.reportDir, creeveyTestId), { recursive: true });
+
+      for (const attachment of result.attachments) {
+        // Only process image attachments
+        if (!attachment.contentType.startsWith('image/')) continue;
+
+        try {
+          const imageName = attachment.name || `screenshot-${Date.now()}`;
+          const imagePath = path.join(creeveyTestId, `${imageName}.png`);
+          const fullImagePath = path.join(this.reportDir, imagePath);
+
+          // Ensure directory exists
+          await fs.mkdir(path.dirname(fullImagePath), { recursive: true });
+
+          // Handle either buffer or path-based attachments
+          if (attachment.body) {
+            await fs.writeFile(fullImagePath, attachment.body);
+          } else if (attachment.path) {
+            await fs.copyFile(attachment.path, fullImagePath);
+          }
+
+          // Add to images for the test result
+          images[imageName] = { actual: `${imageName}.png` };
+          attachmentPaths.push(imagePath);
+
+          this.logDebug(`Saved screenshot: ${imageName} for test: ${test.title}`);
+        } catch (error) {
+          this.logError(`Failed to process attachment: ${error instanceof Error ? error.message : String(error)}`);
+        }
+      }
+    }
+
+    // Update test status and result
+    const testResult: CreeveyTestResult = {
+      status: status === 'success' ? 'success' : 'failed',
+      retries: result.retry,
+      images,
+      error: result.error?.message ?? undefined,
+      duration: result.duration,
+      attachments: attachmentPaths,
+      browserName: test.parent.project()?.name ?? 'unknown',
+    };
+
+    this.testsManager.updateTestStatus(creeveyTestId, status, testResult);
+  }
+
+  /**
+   * Logs a debug message if debug mode is enabled
+   * @param message Message to log
+   */
+  private logDebug(message: string): void {
+    if (this.debug) {
+      console.log(`[Creevey Reporter] ${message}`);
+    }
+  }
+
+  /**
+   * Logs an error message
+   * @param message Error message to log
+   */
+  private logError(message: string): void {
+    console.error(`[Creevey Reporter] ERROR: ${message}`);
+  }
+}