creevey 0.10.0-beta.37 → 0.10.0-beta.39

package/src/server/master/runner.ts

@@ -11,9 +11,16 @@ import {
   TestStatus,
   ServerTest,
   TestMeta,
+  TEST_EVENTS,
+  FakeSuite,
+  FakeTest,
 } from '../../types.js';
 import Pool from './pool.js';
 import { WorkerQueue } from './queue.js';
+import { getTestPath } from '../utils.js';
+
+// NOTE: This is workaround to fix parallel tests running with mocha-junit-reporter
+let isJUnit = false;
 
 export default class Runner extends EventEmitter {
   private failFast: boolean;
@@ -22,6 +29,8 @@ export default class Runner extends EventEmitter {
   private browsers: string[];
   private scheduler: WorkerQueue;
   private pools: Record<string, Pool> = {};
+  private fakeRunner: EventEmitter;
+  private config: Config;
   tests: Partial<Record<string, ServerTest>> = {};
   public get isRunning(): boolean {
     return Object.values(this.pools).some((pool) => pool.isRunning);
@@ -29,11 +38,24 @@ export default class Runner extends EventEmitter {
   constructor(config: Config, gridUrl?: string) {
     super();
 
+    this.config = config;
     this.failFast = config.failFast;
     this.screenDir = config.screenDir;
     this.reportDir = config.reportDir;
     this.scheduler = new WorkerQueue(config.useWorkerQueue);
     this.browsers = Object.keys(config.browsers);
+
+    class FakeRunner extends EventEmitter {}
+    const runner = new FakeRunner();
+    const Reporter = config.reporter;
+
+    if (Reporter.name == 'MochaJUnitReporter') {
+      isJUnit = true;
+    }
+
+    new Reporter(runner, { reporterOptions: config.reporterOptions });
+    this.fakeRunner = runner;
+
     this.browsers
       .map((browser) => (this.pools[browser] = new Pool(this.scheduler, config, browser, gridUrl)))
       .map((pool) => pool.on('test', this.handlePoolMessage));
@@ -45,10 +67,38 @@ export default class Runner extends EventEmitter {
 
     if (!test) return;
     const { browser, testName, storyPath, storyId } = test;
+
+    const fakeSuite: FakeSuite = {
+      title: test.storyPath.slice(0, -1).join('/'),
+      fullTitle: () => fakeSuite.title,
+      titlePath: () => [fakeSuite.title],
+      tests: [],
+    };
+
+    const fakeTest: FakeTest = {
+      parent: fakeSuite,
+      title: [test.story.name, testName, browser].filter(isDefined).join('/'),
+      fullTitle: () => getTestPath(test).join('/'),
+      titlePath: () => getTestPath(test),
+      currentRetry: () => result?.retries,
+      retires: () => this.config.maxRetries,
+      slow: () => 1000,
+      creevey: {
+        reportDir: this.reportDir,
+        sessionId: id, // TODO SessionId
+        browserName: browser,
+        willRetry: (result?.retries ?? 0) < this.config.maxRetries,
+        images: result?.images ?? {},
+      },
+    };
+
+    fakeSuite.tests.push(fakeTest);
+
     // TODO Handle 'retrying' status
     test.status = status == 'retrying' ? 'failed' : status;
     if (!result) {
       // NOTE: Running status
+      this.fakeRunner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
       this.sendUpdate({ tests: { [id]: { id, browser, testName, storyPath, status: test.status, storyId } } });
       return;
     }
@@ -59,6 +109,32 @@ export default class Runner extends EventEmitter {
       test.approved = null;
     }
 
+    const { duration, attachments } = result;
+
+    fakeTest.duration = duration;
+    fakeTest.attachments = attachments;
+    fakeTest.state = result.status === 'failed' ? 'failed' : 'passed';
+    if (duration !== undefined) {
+      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
+    }
+
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_BEGIN, fakeSuite);
+    }
+
+    if (result.status === 'failed') {
+      fakeTest.err = result.error;
+      this.fakeRunner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, result.error);
+    } else {
+      this.fakeRunner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
+    }
+
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_END, fakeSuite);
+    }
+
+    this.fakeRunner.emit(TEST_EVENTS.TEST_END, fakeTest);
+
     this.sendUpdate({
       tests: {
         [id]: {
@@ -79,6 +155,7 @@ export default class Runner extends EventEmitter {
 
   private handlePoolStop = (): void => {
     if (!this.isRunning) {
+      this.fakeRunner.emit(TEST_EVENTS.RUN_END);
       this.sendUpdate({ isRunning: false });
       this.emit('stop');
     }
@@ -148,6 +225,7 @@ export default class Runner extends EventEmitter {
       };
     }, {});
 
+    this.fakeRunner.emit(TEST_EVENTS.RUN_BEGIN);
     this.browsers.forEach((browser) => {
       const pool = this.pools[browser];
       const tests = testsByBrowser[browser];

package/src/server/reporter.ts

@@ -4,14 +4,6 @@ import prefix from 'loglevel-plugin-prefix';
 import { FakeTest, Images, isDefined, isImageError, TEST_EVENTS } from '../types.js';
 import EventEmitter from 'events';
 
-interface ReporterOptions {
-  reportDir: string;
-  sessionId: string;
-  browserName: string;
-  willRetry: boolean;
-  images: Partial<Record<string, Partial<Images>>>;
-}
-
 const testLevels: Record<string, string> = {
   INFO: chalk.green('PASS'),
   WARN: chalk.yellow('START'),
@@ -19,36 +11,43 @@ const testLevels: Record<string, string> = {
 };
 
 export class CreeveyReporter {
+  private logger: Logger.Logger | null = null;
   // TODO Output in better way, like vitest, maybe
-  constructor(runner: EventEmitter, options: { reporterOptions: { creevey: ReporterOptions } }) {
-    const { sessionId, browserName } = options.reporterOptions.creevey;
-    const testLogger = Logger.getLogger(sessionId);
-
-    prefix.apply(testLogger, {
-      format(level) {
-        return `[${browserName}:${chalk.gray(process.pid)}] ${testLevels[level]} => ${chalk.gray(sessionId)}`;
-      },
-    });
-
+  constructor(runner: EventEmitter) {
     runner.on(TEST_EVENTS.TEST_BEGIN, (test: FakeTest) => {
-      testLogger.warn(chalk.cyan(test.fullTitle()));
+      this.getLogger(test.creevey).warn(chalk.cyan(test.fullTitle()));
     });
     runner.on(TEST_EVENTS.TEST_PASS, (test: FakeTest) => {
-      testLogger.info(chalk.cyan(test.fullTitle()), chalk.gray(`(${test.duration} ms)`));
+      this.getLogger(test.creevey).info(chalk.cyan(test.fullTitle()), chalk.gray(`(${test.duration} ms)`));
     });
     runner.on(TEST_EVENTS.TEST_FAIL, (test: FakeTest, error) => {
-      testLogger.error(
+      this.getLogger(test.creevey).error(
         chalk.cyan(test.fullTitle()),
         chalk.gray(`(${test.duration} ms)`),
         '\n  ',
         this.getErrors(
           error,
-          (error, imageName) => `${chalk.bold(imageName ?? browserName)}:${error}`,
+          (error, imageName) => `${chalk.bold(imageName ?? test.creevey.browserName)}:${error}`,
           (error) => error.stack ?? error.message,
         ).join('\n  '),
       );
     });
   }
+
+  private getLogger(options: { sessionId: string; browserName: string }) {
+    if (this.logger) return this.logger;
+    const { sessionId, browserName } = options;
+    const testLogger = Logger.getLogger(sessionId);
+
+    this.logger = prefix.apply(testLogger, {
+      format(level) {
+        return `[${browserName}:${chalk.gray(process.pid)}] ${testLevels[level]} => ${chalk.gray(sessionId)}`;
+      },
+    });
+
+    return this.logger;
+  }
+
   private getErrors(
     error: unknown,
     imageErrorToString: (error: string, imageName?: string) => string,
@@ -72,10 +71,7 @@ export class CreeveyReporter {
 }
 
 export class TeamcityReporter {
-  constructor(runner: EventEmitter, options: { reporterOptions: { creevey: ReporterOptions } }) {
-    const browserName = this.escape(options.reporterOptions.creevey.browserName);
-    const reporterOptions = options.reporterOptions.creevey;
-
+  constructor(runner: EventEmitter) {
     runner.on(TEST_EVENTS.TEST_BEGIN, (test: FakeTest) => {
       console.log(`##teamcity[testStarted name='${this.escape(test.fullTitle())}' flowId='${process.pid}']`);
     });
@@ -85,7 +81,8 @@ export class TeamcityReporter {
     });
 
     runner.on(TEST_EVENTS.TEST_FAIL, (test: FakeTest, error: Error) => {
-      Object.entries(reporterOptions.images).forEach(([name, image]) => {
+      const browserName = this.escape(test.creevey.browserName);
+      Object.entries(test.creevey.images).forEach(([name, image]) => {
         if (!image) return;
         const filePath = test
           .titlePath()
@@ -99,7 +96,7 @@ export class TeamcityReporter {
           .filter(isDefined)
           .forEach((fileName) => {
             console.log(
-              `##teamcity[publishArtifacts '${reporterOptions.reportDir}/${filePath}/${fileName} => report/${filePath}']`,
+              `##teamcity[publishArtifacts '${test.creevey.reportDir}/${filePath}/${fileName} => report/${filePath}']`,
             );
             console.log(
               `##teamcity[testMetadata testName='${this.escape(
@@ -112,7 +109,7 @@ export class TeamcityReporter {
       // Output failed test as passed due TC don't support retry mechanic
       // https://teamcity-support.jetbrains.com/hc/en-us/community/posts/207216829-Count-test-as-successful-if-at-least-one-try-is-successful?page=1#community_comment_207394125
 
-      if (reporterOptions.willRetry)
+      if (test.creevey.willRetry)
         console.log(`##teamcity[testFinished name='${this.escape(test.fullTitle())}' flowId='${process.pid}']`);
       else
         console.log(

package/src/server/worker/start.ts

@@ -1,16 +1,12 @@
 import chai from 'chai';
-import EventEmitter from 'events';
 import {
   BaseCreeveyTestContext,
   Config,
   CreeveyWebdriver,
-  FakeSuite,
-  FakeTest,
-  Images,
   Options,
   ServerTest,
-  TEST_EVENTS,
   TestMessage,
+  TestResult,
   isDefined,
   isImageError,
 } from '../../types.js';
@@ -45,9 +41,10 @@ async function getTestsFromStories(
   return testsById;
 }
 
-function runHandler(browserName: string, images: Partial<Record<string, Images>>, error?: unknown): void {
+function runHandler(browserName: string, result: Omit<TestResult, 'status'>, error?: unknown): void {
   // TODO How handle browser corruption?
-  if (isImageError(error)) {
+  const { images } = result;
+  if (images != null && isImageError(error)) {
     if (typeof error.images == 'string') {
       const image = images[browserName];
       if (image) image.error = error.images;
@@ -60,25 +57,31 @@ function runHandler(browserName: string, images: Partial<Record<string, Images>>
     }
   }
 
-  if (error || Object.values(images).some((image) => image?.error != null)) {
-    const errorMessage = serializeError(error);
+  if (error || (images != null && Object.values(images).some((image) => image?.error != null))) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const errorMessage = result.error!;
 
     const isUnexpectedError =
       hasTimeout(errorMessage) ||
       hasDisconnected(errorMessage) ||
-      Object.values(images).some((image) => hasTimeout(image?.error));
+      (images != null && Object.values(images).some((image) => hasTimeout(image?.error)));
     if (isUnexpectedError) emitWorkerMessage({ type: 'error', payload: { subtype: 'unknown', error: errorMessage } });
     else
       emitTestMessage({
         type: 'end',
         payload: {
           status: 'failed',
-          images,
-          error: errorMessage,
+          ...result,
         },
       });
   } else {
-    emitTestMessage({ type: 'end', payload: { status: 'success', images } });
+    emitTestMessage({
+      type: 'end',
+      payload: {
+        status: 'success',
+        ...result,
+      },
+    });
   }
 }
 
@@ -112,7 +115,6 @@ function hasTimeout(str: string | null | undefined): boolean {
 }
 
 export async function start(browser: string, gridUrl: string, config: Config, options: Options): Promise<void> {
-  let retries = 0;
   const imagesContext: ImageContext = {
     attachments: [],
     testFullPath: [],
@@ -123,26 +125,6 @@ export async function start(browser: string, gridUrl: string, config: Config, op
 
   if (!webdriver || !sessionId) return;
 
-  const reporterOptions = {
-    ...config.reporterOptions,
-    creevey: {
-      sessionId,
-      reportDir: config.reportDir,
-      browserName: browser,
-      get willRetry() {
-        return retries < config.maxRetries;
-      },
-      get images() {
-        return imagesContext.images;
-      },
-    },
-  };
-
-  class FakeRunner extends EventEmitter {}
-  const runner = new FakeRunner();
-  const Reporter = config.reporter;
-  new Reporter(runner, { reporterOptions });
-
   const { matchImage, matchImages } = options.odiff
     ? getOdiffMatchers(imagesContext, config)
     : await getMatchers(imagesContext, config);
@@ -196,32 +178,9 @@ export async function start(browser: string, gridUrl: string, config: Config, op
     imagesContext.testFullPath = getTestPath(test);
     imagesContext.images = {};
 
-    retries = message.payload.retries;
     let error = undefined;
 
-    const fakeSuite: FakeSuite = {
-      title: test.storyPath.slice(0, -1).join('/'),
-      fullTitle: () => fakeSuite.title,
-      titlePath: () => [fakeSuite.title],
-      tests: [],
-    };
-
-    const fakeTest: FakeTest = {
-      parent: fakeSuite,
-      title: [test.story.name, test.testName, test.browser].filter(isDefined).join('/'),
-      fullTitle: () => getTestPath(test).join('/'),
-      titlePath: () => getTestPath(test),
-      currentRetry: () => retries,
-      retires: () => config.maxRetries,
-      slow: () => 1000,
-    };
-
-    fakeSuite.tests.push(fakeTest);
-
     void (async () => {
-      runner.emit(TEST_EVENTS.RUN_BEGIN);
-      runner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
-
       let timeout;
       let isRejected = false;
       const start = Date.now();
@@ -241,22 +200,9 @@ export async function start(browser: string, gridUrl: string, config: Config, op
         ]);
       } catch (testError) {
         error = testError;
-        fakeTest.err = error;
       }
       const duration = Date.now() - start;
       clearTimeout(timeout);
-      fakeTest.attachments = imagesContext.attachments;
-      fakeTest.state = error ? 'failed' : 'passed';
-      fakeTest.duration = duration;
-      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
-
-      if (error) {
-        runner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, error);
-      } else {
-        runner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
-      }
-      runner.emit(TEST_EVENTS.TEST_END, fakeTest);
-      runner.emit(TEST_EVENTS.RUN_END);
 
       await webdriver.afterTest(test);
 
@@ -267,7 +213,14 @@ export async function start(browser: string, gridUrl: string, config: Config, op
           payload: { subtype: 'unknown', error: serializeError(error) },
         });
       } else {
-        runHandler(baseContext.browserName, imagesContext.images, error);
+        const result = {
+          images: imagesContext.images,
+          error: serializeError(error),
+          duration,
+          attachments: imagesContext.attachments,
+          retries: message.payload.retries,
+        };
+        runHandler(baseContext.browserName, result, error);
       }
     })().catch((error: unknown) => {
       logger().error('Unexpected error:', error);

package/src/types.ts

@@ -419,10 +419,14 @@ export type TestStatus = 'unknown' | 'pending' | 'running' | 'failed' | 'approve
 
 export interface TestResult {
   status: 'failed' | 'success';
+  retries: number;
   // TODO Remove checks `name == browser` in TestResultsView
   // images?: Partial<{ [name: string]: Images }> | Images;
   images?: Partial<Record<string, Images>>;
   error?: string;
+  // Test metadata for reporting
+  duration?: number;
+  attachments?: string[];
 }
 
 export class ImagesError extends Error {
@@ -470,6 +474,8 @@ export interface CreeveyTestContext extends BaseCreeveyTestContext {
 export enum TEST_EVENTS {
   RUN_BEGIN = 'start',
   RUN_END = 'end',
+  SUITE_BEGIN = 'suite',
+  SUITE_END = 'suite end',
   TEST_BEGIN = 'test',
   TEST_END = 'test end',
   TEST_FAIL = 'fail',
@@ -494,7 +500,7 @@ export interface FakeTest {
   title: string;
   fullTitle: () => string;
   titlePath: () => string[];
-  currentRetry: () => number;
+  currentRetry: () => number | undefined;
   retires: () => number;
   slow: () => number;
   duration?: number;
@@ -504,6 +510,15 @@ export interface FakeTest {
   err?: unknown;
   // NOTE: image files
   attachments?: string[];
+
+  // NOTE: Creevey specific fields
+  creevey: {
+    reportDir: string;
+    sessionId: string;
+    browserName: string;
+    willRetry: boolean;
+    images: Partial<Record<string, Partial<Images>>>;
+  };
 }
 
 export interface CreeveyStatus {