creevey 0.10.0-beta.9 → 0.10.0-rc.1

package/src/server/master/queue.ts

@@ -8,14 +8,20 @@ type MaybeWorker = Worker | { error: string };
 
 export class WorkerQueue {
   private isProcessing = false;
-  private queue: { browser: string; gridUrl?: string; retry: number; resolve: (mw: MaybeWorker) => void }[] = [];
+  private queue: {
+    browser: string;
+    storybookUrl: string;
+    gridUrl?: string;
+    retry: number;
+    resolve: (mw: MaybeWorker) => void;
+  }[] = [];
 
   // TODO Add concurrency
   constructor(private useQueue: boolean) {}
 
-  async forkWorker(browser: string, gridUrl?: string, retry = 0): Promise<MaybeWorker> {
+  async forkWorker(browser: string, storybookUrl: string, gridUrl?: string, retry = 0): Promise<MaybeWorker> {
     return new Promise<MaybeWorker>((resolve) => {
-      this.queue.push({ browser, gridUrl, retry, resolve });
+      this.queue.push({ browser, storybookUrl, gridUrl, retry, resolve });
 
       void this.process();
     });
@@ -24,9 +30,9 @@ export class WorkerQueue {
   private async process() {
     if (this.useQueue && this.isProcessing) return;
 
-    const { browser, gridUrl, retry, resolve } = this.queue.pop() ?? {};
+    const { browser, storybookUrl, gridUrl, retry, resolve } = this.queue.pop() ?? {};
 
-    if (browser == undefined || retry == undefined || resolve == undefined) return;
+    if (browser == undefined || storybookUrl == undefined || retry == undefined || resolve == undefined) return;
 
     if (isShuttingDown.current) {
       resolve({ error: 'Master process is shutting down' });
@@ -36,7 +42,15 @@ export class WorkerQueue {
     this.isProcessing = true;
 
     cluster.setupPrimary({
-      args: ['--browser', browser, ...(gridUrl ? ['--gridUrl', gridUrl] : []), ...process.argv.slice(2)],
+      args: [
+        'worker',
+        '--browser',
+        browser,
+        ...(gridUrl ? ['--gridUrl', gridUrl] : []),
+        ...process.argv.slice(3),
+        '--storybookUrl',
+        storybookUrl,
+      ],
     });
     const worker = cluster.fork();
     const message = await new Promise((resolve: (value: WorkerMessage) => void) => {
@@ -52,7 +66,7 @@ export class WorkerQueue {
       gracefullyKill(worker);
 
       if (retry == FORK_RETRIES) resolve(message.payload);
-      else this.queue.push({ browser, gridUrl, retry: retry + 1, resolve });
+      else this.queue.push({ browser, storybookUrl, gridUrl, retry: retry + 1, resolve });
     } else {
       resolve(worker);
     }

package/src/server/master/runner.ts

@@ -1,5 +1,3 @@
-import path from 'path';
-import { copyFile, mkdir } from 'fs/promises';
 import { EventEmitter } from 'events';
 import {
   Config,
@@ -10,30 +8,59 @@ import {
   CreeveyUpdate,
   TestStatus,
   ServerTest,
-  TestMeta,
+  TEST_EVENTS,
+  FakeSuite,
+  FakeTest,
 } from '../../types.js';
 import Pool from './pool.js';
 import { WorkerQueue } from './queue.js';
+import { getTestPath } from '../utils.js';
+import { getReporter } from '../reporters/index.js';
+import type { TestsManager } from './testsManager.js';
+
+// NOTE: This is workaround to fix parallel tests running with mocha-junit-reporter
+let isJUnit = false;
+
+class FakeRunner extends EventEmitter {
+  public stats = {
+    duration: 0,
+    failures: 0,
+    pending: 0,
+  };
+}
 
 export default class Runner extends EventEmitter {
   private failFast: boolean;
-  private screenDir: string;
-  private reportDir: string;
   private browsers: string[];
   private scheduler: WorkerQueue;
   private pools: Record<string, Pool> = {};
-  tests: Partial<Record<string, ServerTest>> = {};
+  private fakeRunner: FakeRunner;
+  private config: Config;
+  public testsManager: TestsManager;
+
   public get isRunning(): boolean {
     return Object.values(this.pools).some((pool) => pool.isRunning);
   }
-  constructor(config: Config, gridUrl?: string) {
+
+  constructor(config: Config, testsManager: TestsManager, gridUrl?: string) {
     super();
 
+    this.config = config;
     this.failFast = config.failFast;
-    this.screenDir = config.screenDir;
-    this.reportDir = config.reportDir;
+    this.testsManager = testsManager;
     this.scheduler = new WorkerQueue(config.useWorkerQueue);
     this.browsers = Object.keys(config.browsers);
+
+    const runner = new FakeRunner();
+    const Reporter = getReporter(config.reporter);
+
+    if (Reporter.name == 'MochaJUnitReporter') {
+      isJUnit = true;
+    }
+
+    new Reporter(runner, { reportDir: config.reportDir, reporterOptions: config.reporterOptions });
+    this.fakeRunner = runner;
+
     this.browsers
       .map((browser) => (this.pools[browser] = new Pool(this.scheduler, config, browser, gridUrl)))
       .map((pool) => pool.on('test', this.handlePoolMessage));
@@ -41,46 +68,83 @@ export default class Runner extends EventEmitter {
 
   private handlePoolMessage = (message: { id: string; status: TestStatus; result?: TestResult }): void => {
     const { id, status, result } = message;
-    const test = this.tests[id];
+    const test = this.testsManager.getTest(id);
 
     if (!test) return;
-    const { browser, testName, storyPath, storyId } = test;
-    // TODO Handle 'retrying' status
-    test.status = status == 'retrying' ? 'failed' : status;
+    const { browser, testName } = test;
+
+    const fakeSuite: FakeSuite = {
+      title: test.storyPath.slice(0, -1).join('/'),
+      fullTitle: () => fakeSuite.title,
+      titlePath: () => [fakeSuite.title],
+      tests: [],
+    };
+
+    const fakeTest: FakeTest = {
+      parent: fakeSuite,
+      title: [test.story.name, testName, browser].filter(isDefined).join('/'),
+      fullTitle: () => getTestPath(test).join('/'),
+      titlePath: () => getTestPath(test),
+      currentRetry: () => result?.retries,
+      retires: () => this.config.maxRetries,
+      slow: () => 1000,
+      err: result?.error,
+      creevey: {
+        sessionId: result?.sessionId ?? id,
+        browserName: result?.browserName ?? browser,
+        workerId: result?.workerId ?? process.pid,
+        willRetry: (result?.retries ?? 0) < this.config.maxRetries,
+        images: result?.images ?? {},
+      },
+    };
+
+    fakeSuite.tests.push(fakeTest);
+
+    const update = this.testsManager.updateTestStatus(id, status, result);
+    if (!update) return;
+
     if (!result) {
-      // NOTE: Running status
-      this.sendUpdate({ tests: { [id]: { id, browser, testName, storyPath, status: test.status, storyId } } });
+      this.fakeRunner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
+      this.sendUpdate(update);
       return;
     }
-    if (!test.results) {
-      test.results = [];
+
+    const { duration, attachments } = result;
+
+    fakeTest.duration = duration;
+    fakeTest.attachments = attachments;
+    fakeTest.state = result.status === 'failed' ? 'failed' : 'passed';
+    if (duration !== undefined) {
+      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
     }
-    test.results.push(result);
 
-    if (status == 'failed') {
-      test.approved = null;
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_BEGIN, fakeSuite);
     }
 
-    this.sendUpdate({
-      tests: {
-        [id]: {
-          id,
-          browser,
-          testName,
-          storyPath,
-          status: test.status,
-          approved: test.approved,
-          results: [result],
-          storyId,
-        },
-      },
-    });
+    if (result.status === 'failed') {
+      fakeTest.err = result.error;
+      this.fakeRunner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, result.error);
+      this.fakeRunner.stats.failures++;
+    } else {
+      this.fakeRunner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
+      this.fakeRunner.stats.duration += duration ?? 0;
+    }
+
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_END, fakeSuite);
+    }
+
+    this.fakeRunner.emit(TEST_EVENTS.TEST_END, fakeTest);
+
+    this.sendUpdate(update);
 
     if (this.failFast && status == 'failed') this.stop();
   };
 
   private handlePoolStop = (): void => {
     if (!this.isRunning) {
+      this.fakeRunner.emit(TEST_EVENTS.RUN_END);
       this.sendUpdate({ isRunning: false });
       this.emit('stop');
     }
@@ -91,31 +155,8 @@ export default class Runner extends EventEmitter {
   }
 
   public updateTests(testsDiff: Partial<Record<string, ServerTest>>): void {
-    const tests: CreeveyStatus['tests'] = {};
-    const removedTests: TestMeta[] = [];
-    Object.entries(testsDiff).forEach(([id, newTest]) => {
-      const oldTest = this.tests[id];
-      if (newTest) {
-        if (oldTest) {
-          this.tests[id] = {
-            ...newTest,
-            retries: oldTest.retries,
-            results: oldTest.results,
-            approved: oldTest.approved,
-          };
-        } else this.tests[id] = newTest;
-
-        const { story: _, fn: __, ...restTest } = newTest;
-        tests[id] = { ...restTest, status: 'unknown' };
-      } else if (oldTest) {
-        const { id, browser, testName, storyPath, storyId } = oldTest;
-        removedTests.push({ id, browser, testName, storyPath, storyId });
-        // TODO Use Map instead
-        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-        delete this.tests[id];
-      }
-    });
-    this.sendUpdate({ tests, removedTests });
+    const update = this.testsManager.updateTests(testsDiff);
+    if (update) this.sendUpdate(update);
   }
 
   public start(ids: string[]): void {
@@ -123,33 +164,39 @@ export default class Runner extends EventEmitter {
     if (this.isRunning) return;
 
     const testsToStart = ids
-      .map((id) => this.tests[id])
+      .map((id) => this.testsManager.getTest(id))
       .filter(isDefined)
       .filter((test) => !test.skip);
 
     if (testsToStart.length == 0) return;
 
+    const pendingTests: CreeveyUpdate['tests'] = testsToStart.reduce(
+      (update: CreeveyUpdate['tests'], { id, storyId, browser, testName, storyPath }) => ({
+        ...update,
+        [id]: { id, browser, testName, storyPath, status: 'pending', storyId },
+      }),
+      {},
+    );
+
     this.sendUpdate({
       isRunning: true,
-      tests: testsToStart.reduce(
-        (update: CreeveyUpdate['tests'], { id, storyId, browser, testName, storyPath }) => ({
-          ...update,
-          [id]: { id, browser, testName, storyPath, status: 'pending', storyId },
-        }),
-        {},
-      ),
+      tests: pendingTests,
     });
 
     const testsByBrowser: Partial<TestsByBrowser> = testsToStart.reduce((tests: Partial<TestsByBrowser>, test) => {
       const { id, browser, testName, storyPath } = test;
       const restPath = [...storyPath, testName].filter(isDefined);
-      test.status = 'pending';
+
+      // Update status to pending in TestsManager
+      this.testsManager.updateTestStatus(id, 'pending');
+
       return {
         ...tests,
         [browser]: [...(tests[browser] ?? []), { id, path: restPath }],
       };
     }, {});
 
+    this.fakeRunner.emit(TEST_EVENTS.RUN_BEGIN);
     this.browsers.forEach((browser) => {
       const pool = this.pools[browser];
       const tests = testsByBrowser[browser];
@@ -168,82 +215,24 @@ export default class Runner extends EventEmitter {
   }
 
   public get status(): CreeveyStatus {
-    const tests: CreeveyStatus['tests'] = {};
-    Object.values(this.tests)
-      .filter(isDefined)
-
-      .forEach(({ story: _, fn: __, ...test }) => (tests[test.id] = test));
     return {
       isRunning: this.isRunning,
-      tests,
+      tests: this.testsManager.getTestsData(),
       browsers: this.browsers,
+      isUpdateMode: false,
     };
   }
 
-  private async copyImage(test: ServerTest, image: string, actual: string): Promise<void> {
-    const { browser, testName, storyPath } = test;
-    const restPath = [...storyPath, testName].filter(isDefined);
-    const testPath = path.join(...restPath, image == browser ? '' : browser);
-    const srcImagePath = path.join(this.reportDir, testPath, actual);
-    const dstImagePath = path.join(this.screenDir, testPath, `${image}.png`);
-    await mkdir(path.join(this.screenDir, testPath), { recursive: true });
-    await copyFile(srcImagePath, dstImagePath);
-  }
-
   public async approveAll(): Promise<void> {
-    const updatedTests: NonNullable<CreeveyUpdate['tests']> = {};
-    for (const test of Object.values(this.tests)) {
-      if (!test?.results) continue;
-      const retry = test.results.length - 1;
-      const { images, status } = test.results.at(retry) ?? {};
-      if (!images || status != 'failed') continue;
-      for (const [name, image] of Object.entries(images)) {
-        if (!image) continue;
-        await this.copyImage(test, name, image.actual);
-
-        if (!test.approved) {
-          test.approved = {};
-        }
-        test.approved[name] = retry;
-        test.status = 'approved';
-
-        updatedTests[test.id] = {
-          id: test.id,
-          browser: test.browser,
-          storyPath: test.storyPath,
-          storyId: test.storyId,
-          status: test.status,
-          approved: { [name]: retry },
-        };
-      }
-    }
-    this.sendUpdate({ tests: updatedTests });
+    const update = await this.testsManager.approveAll();
+    this.sendUpdate(update);
   }
 
-  public async approve({ id, retry, image }: ApprovePayload): Promise<void> {
-    const test = this.tests[id];
-    if (!test?.results) return;
-    const result = test.results[retry];
-    if (!result.images) return;
-    const images = result.images[image];
-    if (!images) return;
-    if (!test.approved) {
-      test.approved = {};
-    }
-    const { browser, testName, storyPath, storyId } = test;
-
-    await this.copyImage(test, image, images.actual);
-
-    test.approved[image] = retry;
-
-    if (Object.keys(result.images).every((name) => typeof test.approved?.[name] == 'number')) {
-      test.status = 'approved';
-    }
-
-    this.sendUpdate({
-      tests: { [id]: { id, browser, testName, storyPath, status: test.status, approved: { [image]: retry }, storyId } },
-    });
+  public async approve(payload: ApprovePayload): Promise<void> {
+    const update = await this.testsManager.approve(payload);
+    if (update) this.sendUpdate(update);
   }
+
   private sendUpdate(data: CreeveyUpdate): void {
     this.emit('update', data);
   }