creevey 0.10.0-beta.4 → 0.10.0-beta.40

package/src/server/master/runner.ts

@@ -11,15 +11,34 @@ import {
   TestStatus,
   ServerTest,
   TestMeta,
+  TEST_EVENTS,
+  FakeSuite,
+  FakeTest,
 } from '../../types.js';
 import Pool from './pool.js';
+import { WorkerQueue } from './queue.js';
+import { getTestPath } from '../utils.js';
+
+// NOTE: This is workaround to fix parallel tests running with mocha-junit-reporter
+let isJUnit = false;
+
+class FakeRunner extends EventEmitter {
+  public stats = {
+    duration: 0,
+    failures: 0,
+    pending: 0,
+  };
+}
 
 export default class Runner extends EventEmitter {
   private failFast: boolean;
   private screenDir: string;
   private reportDir: string;
   private browsers: string[];
+  private scheduler: WorkerQueue;
   private pools: Record<string, Pool> = {};
+  private fakeRunner: FakeRunner;
+  private config: Config;
   tests: Partial<Record<string, ServerTest>> = {};
   public get isRunning(): boolean {
     return Object.values(this.pools).some((pool) => pool.isRunning);
@@ -27,12 +46,25 @@ export default class Runner extends EventEmitter {
   constructor(config: Config, gridUrl?: string) {
     super();
 
+    this.config = config;
     this.failFast = config.failFast;
     this.screenDir = config.screenDir;
     this.reportDir = config.reportDir;
+    this.scheduler = new WorkerQueue(config.useWorkerQueue);
     this.browsers = Object.keys(config.browsers);
+
+    const runner = new FakeRunner();
+    const Reporter = config.reporter;
+
+    if (Reporter.name == 'MochaJUnitReporter') {
+      isJUnit = true;
+    }
+
+    new Reporter(runner, { reporterOptions: config.reporterOptions });
+    this.fakeRunner = runner;
+
     this.browsers
-      .map((browser) => (this.pools[browser] = new Pool(config, browser, gridUrl)))
+      .map((browser) => (this.pools[browser] = new Pool(this.scheduler, config, browser, gridUrl)))
       .map((pool) => pool.on('test', this.handlePoolMessage));
   }
 
@@ -42,22 +74,76 @@ export default class Runner extends EventEmitter {
 
     if (!test) return;
     const { browser, testName, storyPath, storyId } = test;
+
+    const fakeSuite: FakeSuite = {
+      title: test.storyPath.slice(0, -1).join('/'),
+      fullTitle: () => fakeSuite.title,
+      titlePath: () => [fakeSuite.title],
+      tests: [],
+    };
+
+    const fakeTest: FakeTest = {
+      parent: fakeSuite,
+      title: [test.story.name, testName, browser].filter(isDefined).join('/'),
+      fullTitle: () => getTestPath(test).join('/'),
+      titlePath: () => getTestPath(test),
+      currentRetry: () => result?.retries,
+      retires: () => this.config.maxRetries,
+      slow: () => 1000,
+      creevey: {
+        reportDir: this.reportDir,
+        sessionId: result?.sessionId ?? id,
+        browserName: browser,
+        willRetry: (result?.retries ?? 0) < this.config.maxRetries,
+        images: result?.images ?? {},
+      },
+    };
+
+    fakeSuite.tests.push(fakeTest);
+
     // TODO Handle 'retrying' status
     test.status = status == 'retrying' ? 'failed' : status;
     if (!result) {
       // NOTE: Running status
+      this.fakeRunner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
       this.sendUpdate({ tests: { [id]: { id, browser, testName, storyPath, status: test.status, storyId } } });
       return;
     }
-    if (!test.results) {
-      test.results = [];
-    }
+    test.results ??= [];
     test.results.push(result);
 
     if (status == 'failed') {
       test.approved = null;
     }
 
+    const { duration, attachments } = result;
+
+    fakeTest.duration = duration;
+    fakeTest.attachments = attachments;
+    fakeTest.state = result.status === 'failed' ? 'failed' : 'passed';
+    if (duration !== undefined) {
+      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
+    }
+
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_BEGIN, fakeSuite);
+    }
+
+    if (result.status === 'failed') {
+      fakeTest.err = result.error;
+      this.fakeRunner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, result.error);
+      this.fakeRunner.stats.failures++;
+    } else {
+      this.fakeRunner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
+      this.fakeRunner.stats.duration += duration ?? 0;
+    }
+
+    if (isJUnit) {
+      this.fakeRunner.emit(TEST_EVENTS.SUITE_END, fakeSuite);
+    }
+
+    this.fakeRunner.emit(TEST_EVENTS.TEST_END, fakeTest);
+
     this.sendUpdate({
       tests: {
         [id]: {
@@ -78,6 +164,7 @@ export default class Runner extends EventEmitter {
 
   private handlePoolStop = (): void => {
     if (!this.isRunning) {
+      this.fakeRunner.emit(TEST_EVENTS.RUN_END);
       this.sendUpdate({ isRunning: false });
       this.emit('stop');
     }
@@ -147,6 +234,7 @@ export default class Runner extends EventEmitter {
       };
     }, {});
 
+    this.fakeRunner.emit(TEST_EVENTS.RUN_BEGIN);
     this.browsers.forEach((browser) => {
       const pool = this.pools[browser];
       const tests = testsByBrowser[browser];
@@ -198,9 +286,7 @@ export default class Runner extends EventEmitter {
         if (!image) continue;
         await this.copyImage(test, name, image.actual);
 
-        if (!test.approved) {
-          test.approved = {};
-        }
+        test.approved ??= {};
         test.approved[name] = retry;
         test.status = 'approved';
 
@@ -224,9 +310,7 @@ export default class Runner extends EventEmitter {
     if (!result.images) return;
     const images = result.images[image];
     if (!images) return;
-    if (!test.approved) {
-      test.approved = {};
-    }
+    test.approved ??= {};
     const { browser, testName, storyPath, storyId } = test;
 
     await this.copyImage(test, image, images.actual);

package/src/server/master/server.ts

@@ -96,7 +96,7 @@ export function start(reportDir: string, port: number, ui: boolean): (api: Creev
   app.use(mount('/report', serve(reportDir)));
 
   wss.on('error', (error) => {
-    logger.error(error);
+    logger().error(error);
   });
 
   server.listen(port);

package/src/server/master/start.ts

@@ -14,13 +14,14 @@ import { sendScreenshotsCount } from '../telemetry.js';
 const importMetaUrl = pathToFileURL(__filename).href;
 
 async function copyStatics(reportDir: string): Promise<void> {
-  const clientDir = path.join(path.dirname(fileURLToPath(importMetaUrl)), '../../client/web');
-  const files = (await readdir(clientDir, { withFileTypes: true }))
-    .filter((dirent) => dirent.isFile() && !dirent.name.endsWith('.d.ts') && !dirent.name.endsWith('.tsx'))
+  const clientDir = path.join(path.dirname(fileURLToPath(importMetaUrl)), '../../../dist/client/web');
+  const assets = (await readdir(path.join(clientDir, 'assets'), { withFileTypes: true }))
+    .filter((dirent) => dirent.isFile())
     .map((dirent) => dirent.name);
-  await mkdir(reportDir, { recursive: true });
-  for (const file of files) {
-    await copyFile(path.join(clientDir, file), path.join(reportDir, file));
+  await mkdir(path.join(reportDir, 'assets'), { recursive: true });
+  await copyFile(path.join(clientDir, 'index.html'), path.join(reportDir, 'index.html'));
+  for (const asset of assets) {
+    await copyFile(path.join(clientDir, 'assets', asset), path.join(reportDir, 'assets', asset));
   }
 }
 
@@ -42,7 +43,10 @@ function outputUnnecessaryImages(imagesDir: string, images: Set<string>): void {
     .map((imagePath) => path.posix.relative(imagesDir, imagePath))
     .filter((imagePath) => !images.has(imagePath));
   if (unnecessaryImages.length > 0) {
-    logger.warn('We found unnecessary screenshot images, those can be safely removed:\n', unnecessaryImages.join('\n'));
+    logger().warn(
+      'We found unnecessary screenshot images, those can be safely removed:\n',
+      unnecessaryImages.join('\n'),
+    );
   }
 }
 
@@ -81,10 +85,10 @@ export async function start(
 
   if (options.ui) {
     resolveApi(creeveyApi(runner));
-    logger.info(`Started on http://localhost:${options.port}`);
+    logger().info(`Started on http://localhost:${options.port}`);
   } else {
     if (Object.values(runner.status.tests).filter((test) => test && !test.skip).length == 0) {
-      logger.warn("Don't have any tests to run");
+      logger().warn("Don't have any tests to run");
 
       void shutdownWorkers().then(() => process.exit());
       return;
@@ -101,10 +105,11 @@ export async function start(
       void sendScreenshotsCount(config, options, runner.status)
         .catch((reason: unknown) => {
           const error = reason instanceof Error ? (reason.stack ?? reason.message) : (reason as string);
-          logger.warn(`Can't send telemetry: ${error}`);
+          logger().warn(`Can't send telemetry: ${error}`);
         })
         .finally(() => {
-          void shutdownWorkers().then(() => process.exit());
+          // NOTE: Take some time to kill processes
+          void shutdownWorkers().then(() => setTimeout(() => process.exit(), 500));
         });
     });
     // TODO grep

package/src/server/playwright/docker-file.ts

@@ -1,8 +1,12 @@
+import { readFile } from 'fs/promises';
+import { pathToFileURL } from 'url';
 import semver from 'semver';
 import { exec } from 'shelljs';
 
+const importMetaUrl = pathToFileURL(__filename).href;
+
 // TODO Support custom docker images
-export function playwrightDockerFile(browser: string, version: string): string {
+export async function playwrightDockerFile(browser: string, version: string): Promise<string> {
   const sv = semver.coerce(version);
 
   let npmRegistry;
@@ -12,23 +16,31 @@ export function playwrightDockerFile(browser: string, version: string): string {
     /* noop */
   }
 
-  return `
-FROM mcr.microsoft.com/playwright:v${sv?.format() ?? version}
+  const indexJs = await readFile(new URL('./index-source.mjs', importMetaUrl), 'utf-8');
+
+  const dockerfile = `
+FROM node:lts
 
 WORKDIR /creevey
 
 RUN echo "{ \\"type\\": \\"module\\" }" > package.json && \\
-    echo "import { ${browser} as browser } from 'playwright-core';" >> index.js && \\
-    echo "const ws = await browser.launchServer({ port: 4444, wsPath: 'creevey' })" >> index.js && \\${
+${indexJs
+  .split('\n')
+  .map((line) => `    echo "${line.replace(/"/g, '\\"')}" >> index.js && \\`)
+  .join('\n')}
+    ${
       npmRegistry
         ? `
     echo "registry=${npmRegistry}" > .npmrc && \\`
         : ''
     }
-    npm i playwright-core${sv ? `@${sv.format()}` : ''}
+    npm i playwright-core${sv ? `@${sv.format()}` : ''} && \\
+    npx -y playwright${sv ? `@${sv.format()}` : ''} install --with-deps ${browser}
 
 EXPOSE 4444
 
 ENTRYPOINT [ "node", "./index.js" ]
 `;
+
+  return dockerfile.replace(/\\\n\s*\\?\n/g, '\\\n');
 }

package/src/server/playwright/docker.ts

@@ -1,9 +1,17 @@
+import assert from 'assert';
 import { runImage } from '../docker';
 import { emitWorkerMessage, subscribeOn } from '../messages';
-import { isInsideDocker } from '../utils';
+import { getCreeveyCache, isInsideDocker, resolvePlaywrightBrowserType } from '../utils';
 import { LOCALHOST_REGEXP } from '../webdriver';
+import type { BrowserConfigObject, Config } from '../../types';
 
-export async function startPlaywrightContainer(imageName: string, debug: boolean): Promise<string> {
+export async function startPlaywrightContainer(
+  imageName: string,
+  browser: string,
+  config: Config,
+  debug: boolean,
+): Promise<string> {
+  const { browserName, playwrightOptions } = config.browsers[browser] as BrowserConfigObject;
   const port = await new Promise<number>((resolve) => {
     subscribeOn('worker', (message) => {
       if (message.type == 'port') {
@@ -13,13 +21,18 @@ export async function startPlaywrightContainer(imageName: string, debug: boolean
     emitWorkerMessage({ type: 'port', payload: { port: -1 } });
   });
 
+  const cacheDir = await getCreeveyCache();
+
+  assert(cacheDir, "Couldn't get cache directory");
+
   const host = await runImage(
     imageName,
-    [],
+    [JSON.stringify({ ...playwrightOptions, browser: resolvePlaywrightBrowserType(browserName) })],
     {
       ExposedPorts: { [`${port}/tcp`]: {} },
       HostConfig: {
         PortBindings: { ['4444/tcp']: [{ HostPort: `${port}` }] },
+        Binds: [`${cacheDir}/${process.pid}:/creevey/traces`],
       },
     },
     debug,

package/src/server/playwright/index-source.mjs

@@ -0,0 +1,16 @@
+import { chromium, firefox, webkit } from 'playwright-core';
+
+/** @type import("playwright-core").LaunchOptions & { browser: 'chromium' | 'firefox' | 'webkit' } */
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const config = JSON.parse(process.argv.slice(2)[0]);
+
+const browsers = { chromium, firefox, webkit };
+
+const ws = await browsers[config.browser].launchServer({
+  ...config,
+  port: 4444,
+  wsPath: 'creevey',
+  tracesDir: 'traces',
+});
+
+console.log(config.browser, 'browser server launched on:', ws.wsEndpoint());