creevey 0.10.0-beta.0 → 0.10.0-beta.10

package/src/selenium.ts

@@ -0,0 +1 @@
+export { SeleniumWebdriver } from './server/selenium/webdriver.js';

package/src/server/config.ts

@@ -5,12 +5,13 @@ import { loadStories as browserStoriesProvider } from './providers/browser.js';
 import { Config, BrowserConfig, BrowserConfigObject, Options, isDefined } from '../types.js';
 import { configExt, loadThroughTSX } from './utils.js';
 import { CreeveyReporter, TeamcityReporter } from './reporter.js';
-import { SeleniumWebdriver } from './selenium/webdriver.js';
+import { logger } from './logger.js';
 
 export const defaultBrowser = 'chrome';
 
-export const defaultConfig: Omit<Config, 'gridUrl' | 'testsDir' | 'tsConfig'> = {
+export const defaultConfig: Omit<Config, 'gridUrl' | 'testsDir' | 'tsConfig' | 'webdriver'> = {
   disableTelemetry: false,
+  useWorkerQueue: false,
   useDocker: true,
   dockerImage: 'aerokube/selenoid:latest-release',
   dockerImagePlatform: '',
@@ -21,11 +22,10 @@ export const defaultConfig: Omit<Config, 'gridUrl' | 'testsDir' | 'tsConfig'> =
   reportDir: path.resolve('report'),
   reporter: process.env.TEAMCITY_VERSION ? TeamcityReporter : CreeveyReporter,
   storiesProvider: browserStoriesProvider,
-  webdriver: SeleniumWebdriver,
   maxRetries: 0,
   testTimeout: 30000,
-  diffOptions: { threshold: 0.05, includeAA: false },
-  odiffOptions: { threshold: 0.05, antialiasing: true },
+  diffOptions: { threshold: 0.1, includeAA: false },
+  odiffOptions: { threshold: 0.1, antialiasing: true },
   browsers: { [defaultBrowser]: true },
   hooks: {},
   testsRegex: /\.creevey\.(t|j)s$/,
@@ -62,11 +62,23 @@ export async function readConfig(options: Options): Promise<Config> {
   const userConfig: typeof defaultConfig & Partial<Pick<Config, 'gridUrl' | 'storiesProvider'>> = { ...defaultConfig };
 
   if (isDefined(configPath)) {
-    const configModule = await loadThroughTSX<{ default: Partial<Config> } | Partial<Config>>((load) => {
+    const configModule = await loadThroughTSX<
+      { default: { default: Partial<Config> } | Partial<Config> } | Partial<Config>
+    >((load) => {
       const configFileUrl = pathToFileURL(configPath).toString();
       return load(configFileUrl);
     });
-    const configData = 'default' in configModule ? configModule.default : configModule;
+    let configData = 'default' in configModule ? configModule.default : configModule;
+    // NOTE In node > 18 with commonjs project and esm config with tsconfig moduleResolution nodeNext there is additional 'default'
+    configData = 'default' in configData ? configData.default : configData;
+
+    if (!configData.webdriver) {
+      const { SeleniumWebdriver } = await import('./selenium/webdriver.js');
+      logger().warn(
+        "Creevey supports `Selenium` and `Playwright` webdrivers. For backward compatibility `Selenium` is used by default, but it might changed in the future. Please explicitly specify one of webdrivers in your Creevey's config",
+      );
+      configData.webdriver = SeleniumWebdriver;
+    }
 
     Object.assign(userConfig, configData);
   }

package/src/server/docker.ts

@@ -21,7 +21,7 @@ export async function pullImages(
   if (auth) args.authconfig = auth;
   if (platform) args.platform = platform;
 
-  logger.info('Pull docker images');
+  logger().info('Pull docker images');
   // TODO Replace with `import from`
   const { default: yoctoSpinner } = await import('yocto-spinner');
   for (const image of images) {
@@ -61,7 +61,7 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
   const images = await docker.listImages({ filters: { label: [`creevey=${imageName}`] } });
 
   if (images.at(0)) {
-    logger.info(`Image ${imageName} already exists`);
+    logger().info(`Image ${imageName} already exists`);
     return;
   }
 

package/src/server/index.ts

@@ -1,4 +1,5 @@
 import cluster from 'cluster';
+import path from 'path';
 import { readConfig, defaultBrowser } from './config.js';
 import { Options, Config, BrowserConfigObject, isWorkerMessage } from '../types.js';
 import { logger } from './logger.js';
@@ -6,11 +7,13 @@ import { SeleniumWebdriver } from './selenium/webdriver.js';
 import { LOCALHOST_REGEXP } from './webdriver.js';
 import { isInsideDocker } from './utils.js';
 import { sendWorkerMessage } from './messages.js';
+import { playwrightDockerFile } from './playwright/docker-file.js';
+import { buildImage } from './docker.js';
+import { mkdir, writeFile } from 'fs/promises';
 
 async function startWebdriverServer(browser: string, config: Config, options: Options): Promise<string | undefined> {
   if (config.webdriver === SeleniumWebdriver) {
     if (cluster.isPrimary) {
-      // TODO Get random free port
       const { startSelenoidContainer, startSelenoidStandalone } = await import('./selenium/selenoid.js');
       const gridUrl = 'http://localhost:4444/wd/hub';
       if (config.useDocker) {
@@ -22,17 +25,36 @@ async function startWebdriverServer(browser: string, config: Config, options: Op
     }
     // TODO Worker might want to use docker image of browser or start standalone selenium
   } else {
+    if (config.gridUrl) return undefined;
+
     // TODO start standalone playwright server (useDocker == false)
+    const {
+      default: { version },
+    } = await import('playwright-core/package.json', { with: { type: 'json' } });
 
     if (cluster.isWorker) {
       // TODO Re-use dockerImage
 
+      // TODO Use https://hub.docker.com/r/playwright/chrome
+      // NOTE It will be possible to use `chrome` browserName
       const { startPlaywrightContainer } = await import('./playwright/docker.js');
       const { browserName } = config.browsers[browser] as BrowserConfigObject;
-      const host = await startPlaywrightContainer(browserName, options.debug);
+
+      const imageName = `creevey/${browserName}:v${version}`;
+      const host = await startPlaywrightContainer(imageName, options.debug);
 
       return host;
     } else {
+      const browsers = [...new Set(Object.values(config.browsers).map((c) => (c as BrowserConfigObject).browserName))];
+      await Promise.all(
+        browsers.map(async (browserName) => {
+          const imageName = `creevey/${browserName}:v${version}`;
+          const dockerfile = playwrightDockerFile(browserName, version);
+
+          await buildImage(imageName, dockerfile);
+        }),
+      );
+
       const { default: getPort } = await import('get-port');
 
       cluster.on('message', (worker, message: unknown) => {
@@ -59,6 +81,10 @@ export default async function (options: Options): Promise<void> {
   const { browser = defaultBrowser, update, ui, port } = options;
   let gridUrl = cluster.isPrimary ? config.gridUrl : options.gridUrl;
 
+  // TODO Add package.json with `"type": "commonjs"` as workaround for esm packages to load `data.js`
+  await mkdir(config.reportDir, { recursive: true });
+  await writeFile(path.join(config.reportDir, 'package.json'), '{"type": "commonjs"}');
+
   // NOTE: We don't need docker nor selenoid for update option
   if (
     !(gridUrl || (Object.values(config.browsers) as BrowserConfigObject[]).every(({ gridUrl }) => gridUrl)) &&
@@ -73,14 +99,29 @@ export default async function (options: Options): Promise<void> {
       return;
     }
     case cluster.isPrimary: {
-      logger.info('Starting Master Process');
+      if (config.webdriver === SeleniumWebdriver) {
+        try {
+          await import('selenium-webdriver');
+        } catch {
+          logger().error('Failed to start Creevey, missing required dependency: "selenium-webdriver"');
+          process.exit(-1);
+        }
+      } else {
+        try {
+          await import('playwright-core');
+        } catch {
+          logger().error('Failed to start Creevey, missing required dependency: "playwright-core"');
+          process.exit(-1);
+        }
+      }
+      logger().info('Starting Master Process');
 
       const resolveApi = (await import('./master/server.js')).start(config.reportDir, port, ui);
 
       return (await import('./master/start.js')).start(gridUrl, config, options, resolveApi);
     }
     default: {
-      logger.info(`Starting Worker for ${browser}`);
+      logger().info(`Starting Worker for ${browser}`);
 
       // NOTE: We assume that we pass `gridUrl` to worker CLI options
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion

package/src/server/logger.ts

@@ -10,12 +10,16 @@ export const colors = {
   ERROR: chalk.red,
 };
 
+let rootName = 'Creevey';
+
 prefix.reg(Logger);
 prefix.apply(Logger, {
-  format(level, name = 'Creevey') {
+  format(level, name = rootName) {
     const levelColor = colors[level.toUpperCase() as keyof typeof colors];
     return `[${name}:${chalk.gray(process.pid)}] ${levelColor(level)} =>`;
   },
 });
 
-export const logger = Logger.getLogger('Creevey');
+export const setRootName = (newName: string) => (rootName = newName);
+
+export const logger = () => Logger.getLogger(rootName);

package/src/server/master/api.ts

@@ -26,7 +26,7 @@ export default function creeveyApi(runner: Runner): CreeveyApi {
 
     handleMessage(ws: WebSocket, message: WebSocket.Data) {
       if (typeof message != 'string') {
-        logger.info('unhandled message', message);
+        logger().info('unhandled message', message);
         return;
       }
 

package/src/server/master/pool.ts

@@ -1,18 +1,9 @@
-import cluster, { Worker as ClusterWorker } from 'cluster';
+import { Worker as ClusterWorker } from 'cluster';
 import { EventEmitter } from 'events';
-import {
-  Worker,
-  Config,
-  TestResult,
-  BrowserConfigObject,
-  WorkerMessage,
-  TestStatus,
-  isWorkerMessage,
-} from '../../types.js';
-import { sendTestMessage, sendShutdownMessage, subscribeOnWorker } from '../messages.js';
-import { isShuttingDown } from '../utils.js';
-
-const FORK_RETRIES = 5;
+import { Worker, Config, TestResult, BrowserConfigObject, TestStatus } from '../../types.js';
+import { sendTestMessage, subscribeOnWorker } from '../messages.js';
+import { gracefullyKill, isShuttingDown } from '../utils.js';
+import { WorkerQueue } from './queue.js';
 
 interface WorkerTest {
   id: string;
@@ -32,6 +23,7 @@ export default class Pool extends EventEmitter {
     return this.workers.length !== this.freeWorkers.length;
   }
   constructor(
+    public scheduler: WorkerQueue,
     config: Config,
     private browser: string,
     gridUrl?: string,
@@ -46,10 +38,11 @@ export default class Pool extends EventEmitter {
 
   async init(): Promise<void> {
     const poolSize = Math.max(1, this.config.limit ?? 1);
-    // TODO Init queue for workers to smooth browser starting load
-    this.workers = (await Promise.all(Array.from({ length: poolSize }).map(() => this.forkWorker()))).filter(
-      (workerOrError): workerOrError is Worker => workerOrError instanceof ClusterWorker,
-    );
+    this.workers = (
+      await Promise.all(
+        Array.from({ length: poolSize }).map(() => this.scheduler.forkWorker(this.browser, this.gridUrl)),
+      )
+    ).filter((workerOrError): workerOrError is Worker => workerOrError instanceof ClusterWorker);
     if (this.workers.length != poolSize)
       throw new Error(`Can't instantiate workers for ${this.browser} due many errors`);
     this.workers.forEach((worker) => {
@@ -66,7 +59,7 @@ export default class Pool extends EventEmitter {
     return true;
   }
 
-  stop(): void {
+  stop() {
     if (!this.isRunning) {
       this.emit('stop');
       return;
@@ -76,7 +69,7 @@ export default class Pool extends EventEmitter {
     this.queue = [];
   }
 
-  process(): void {
+  process() {
     const worker = this.getFreeWorker();
     const test = this.queue.at(0);
 
@@ -99,7 +92,9 @@ export default class Pool extends EventEmitter {
 
     sendTestMessage(worker, { type: 'start', payload: test });
 
-    this.process();
+    setImmediate(() => {
+      this.process();
+    });
   }
 
   private sendStatus(message: { id: string; status: TestStatus; result?: TestResult }): void {
@@ -120,34 +115,12 @@ export default class Pool extends EventEmitter {
     return this.aliveWorkers.filter((worker) => !worker.isRunning);
   }
 
-  private async forkWorker(retry = 0): Promise<Worker | { error: string }> {
-    cluster.setupPrimary({
-      args: ['--browser', this.browser, ...(this.gridUrl ? ['--gridUrl', this.gridUrl] : []), ...process.argv.slice(2)],
-    });
-    const worker = cluster.fork();
-    const message = await new Promise((resolve: (value: WorkerMessage) => void) => {
-      const readyHandler = (message: unknown): void => {
-        if (!isWorkerMessage(message) || message.type == 'port') return;
-        worker.off('message', readyHandler);
-        resolve(message);
-      };
-      worker.on('message', readyHandler);
-    });
-
-    if (message.type != 'error') return worker;
-
-    this.gracefullyKill(worker);
-
-    if (retry == FORK_RETRIES) return message.payload;
-    return this.forkWorker(retry + 1);
-  }
-
   private exitHandler(worker: Worker): void {
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     worker.once('exit', async () => {
       if (isShuttingDown.current) return;
 
-      const workerOrError = await this.forkWorker();
+      const workerOrError = await this.scheduler.forkWorker(this.browser, this.gridUrl);
 
       if (!(workerOrError instanceof ClusterWorker))
         throw new Error(`Can't instantiate worker for ${this.browser} due many errors`);
@@ -158,17 +131,6 @@ export default class Pool extends EventEmitter {
     });
   }
 
-  private gracefullyKill(worker: Worker): void {
-    worker.isShuttingDown = true;
-    const timeout = setTimeout(() => {
-      worker.kill();
-    }, 10000);
-    worker.on('exit', () => {
-      clearTimeout(timeout);
-    });
-    sendShutdownMessage(worker);
-  }
-
   private shouldRetry(test: WorkerTest): boolean {
     return test.retries < this.maxRetries && !this.forcedStop;
   }
@@ -200,7 +162,7 @@ export default class Pool extends EventEmitter {
         });
 
         if (message.payload.subtype == 'unknown') {
-          this.gracefullyKill(worker);
+          gracefullyKill(worker);
         }
 
         this.handleTestResult(worker, test, { status: 'failed', error: message.payload.error });

package/src/server/master/queue.ts

@@ -0,0 +1,64 @@
+import cluster from 'cluster';
+import { isWorkerMessage, Worker, WorkerMessage } from '../../types.js';
+import { gracefullyKill, isShuttingDown } from '../utils.js';
+
+const FORK_RETRIES = 5;
+
+type MaybeWorker = Worker | { error: string };
+
+export class WorkerQueue {
+  private isProcessing = false;
+  private queue: { browser: string; gridUrl?: string; retry: number; resolve: (mw: MaybeWorker) => void }[] = [];
+
+  // TODO Add concurrency
+  constructor(private useQueue: boolean) {}
+
+  async forkWorker(browser: string, gridUrl?: string, retry = 0): Promise<MaybeWorker> {
+    return new Promise<MaybeWorker>((resolve) => {
+      this.queue.push({ browser, gridUrl, retry, resolve });
+
+      void this.process();
+    });
+  }
+
+  private async process() {
+    if (this.useQueue && this.isProcessing) return;
+
+    const { browser, gridUrl, retry, resolve } = this.queue.pop() ?? {};
+
+    if (browser == undefined || retry == undefined || resolve == undefined) return;
+
+    if (isShuttingDown.current) {
+      resolve({ error: 'Master process is shutting down' });
+      return;
+    }
+
+    this.isProcessing = true;
+
+    cluster.setupPrimary({
+      args: ['--browser', browser, ...(gridUrl ? ['--gridUrl', gridUrl] : []), ...process.argv.slice(2)],
+    });
+    const worker = cluster.fork();
+    const message = await new Promise((resolve: (value: WorkerMessage) => void) => {
+      const readyHandler = (message: unknown): void => {
+        if (!isWorkerMessage(message) || message.type == 'port') return;
+        worker.off('message', readyHandler);
+        resolve(message);
+      };
+      worker.on('message', readyHandler);
+    });
+
+    if (message.type == 'error') {
+      gracefullyKill(worker);
+
+      if (retry == FORK_RETRIES) resolve(message.payload);
+      else this.queue.push({ browser, gridUrl, retry: retry + 1, resolve });
+    } else {
+      resolve(worker);
+    }
+
+    this.isProcessing = false;
+
+    setImmediate(() => void this.process());
+  }
+}

package/src/server/master/runner.ts

@@ -13,12 +13,14 @@ import {
   TestMeta,
 } from '../../types.js';
 import Pool from './pool.js';
+import { WorkerQueue } from './queue.js';
 
 export default class Runner extends EventEmitter {
   private failFast: boolean;
   private screenDir: string;
   private reportDir: string;
   private browsers: string[];
+  private scheduler: WorkerQueue;
   private pools: Record<string, Pool> = {};
   tests: Partial<Record<string, ServerTest>> = {};
   public get isRunning(): boolean {
@@ -30,9 +32,10 @@ export default class Runner extends EventEmitter {
     this.failFast = config.failFast;
     this.screenDir = config.screenDir;
     this.reportDir = config.reportDir;
+    this.scheduler = new WorkerQueue(config.useWorkerQueue);
     this.browsers = Object.keys(config.browsers);
     this.browsers
-      .map((browser) => (this.pools[browser] = new Pool(config, browser, gridUrl)))
+      .map((browser) => (this.pools[browser] = new Pool(this.scheduler, config, browser, gridUrl)))
       .map((pool) => pool.on('test', this.handlePoolMessage));
   }
 

package/src/server/master/server.ts

@@ -96,7 +96,7 @@ export function start(reportDir: string, port: number, ui: boolean): (api: Creev
   app.use(mount('/report', serve(reportDir)));
 
   wss.on('error', (error) => {
-    logger.error(error);
+    logger().error(error);
   });
 
   server.listen(port);

package/src/server/master/start.ts

@@ -42,7 +42,10 @@ function outputUnnecessaryImages(imagesDir: string, images: Set<string>): void {
     .map((imagePath) => path.posix.relative(imagesDir, imagePath))
     .filter((imagePath) => !images.has(imagePath));
   if (unnecessaryImages.length > 0) {
-    logger.warn('We found unnecessary screenshot images, those can be safely removed:\n', unnecessaryImages.join('\n'));
+    logger().warn(
+      'We found unnecessary screenshot images, those can be safely removed:\n',
+      unnecessaryImages.join('\n'),
+    );
   }
 }
 
@@ -81,10 +84,10 @@ export async function start(
 
   if (options.ui) {
     resolveApi(creeveyApi(runner));
-    logger.info(`Started on http://localhost:${options.port}`);
+    logger().info(`Started on http://localhost:${options.port}`);
   } else {
     if (Object.values(runner.status.tests).filter((test) => test && !test.skip).length == 0) {
-      logger.warn("Don't have any tests to run");
+      logger().warn("Don't have any tests to run");
 
       void shutdownWorkers().then(() => process.exit());
       return;
@@ -101,7 +104,7 @@ export async function start(
       void sendScreenshotsCount(config, options, runner.status)
         .catch((reason: unknown) => {
           const error = reason instanceof Error ? (reason.stack ?? reason.message) : (reason as string);
-          logger.warn(`Can't send telemetry: ${error}`);
+          logger().warn(`Can't send telemetry: ${error}`);
         })
         .finally(() => {
           void shutdownWorkers().then(() => process.exit());

package/src/server/playwright/docker-file.ts

@@ -1,9 +1,17 @@
 import semver from 'semver';
+import { exec } from 'shelljs';
 
 // TODO Support custom docker images
 export function playwrightDockerFile(browser: string, version: string): string {
   const sv = semver.coerce(version);
 
+  let npmRegistry;
+  try {
+    npmRegistry = exec('npm config get registry', { silent: true }).stdout.trim();
+  } catch {
+    /* noop */
+  }
+
   return `
 FROM mcr.microsoft.com/playwright:v${sv?.format() ?? version}
 
@@ -11,7 +19,12 @@ WORKDIR /creevey
 
 RUN echo "{ \\"type\\": \\"module\\" }" > package.json && \\
     echo "import { ${browser} as browser } from 'playwright-core';" >> index.js && \\
-    echo "const ws = await browser.launchServer({ port: 4444, wsPath: 'creevey' })" >> index.js && \\
+    echo "const ws = await browser.launchServer({ port: 4444, wsPath: 'creevey' })" >> index.js && \\${
+      npmRegistry
+        ? `
+    echo "registry=${npmRegistry}" > .npmrc && \\`
+        : ''
+    }
     npm i playwright-core${sv ? `@${sv.format()}` : ''}
 
 EXPOSE 4444

package/src/server/playwright/docker.ts

@@ -1,19 +1,9 @@
-import { buildImage, runImage } from '../docker';
+import { runImage } from '../docker';
 import { emitWorkerMessage, subscribeOn } from '../messages';
 import { isInsideDocker } from '../utils';
 import { LOCALHOST_REGEXP } from '../webdriver';
-import { playwrightDockerFile } from './docker-file';
-
-export async function startPlaywrightContainer(browserName: string, debug: boolean): Promise<string> {
-  const {
-    default: { version },
-  } = await import('playwright-core/package.json', { with: { type: 'json' } });
-
-  const imageName = `creevey/${browserName}:v${version}`;
-  const dockerfile = playwrightDockerFile(browserName, version);
-
-  await buildImage(imageName, dockerfile);
 
+export async function startPlaywrightContainer(imageName: string, debug: boolean): Promise<string> {
   const port = await new Promise<number>((resolve) => {
     subscribeOn('worker', (message) => {
       if (message.type == 'port') {