creevey 0.9.1 → 0.10.0-beta.0

package/src/server/config.ts

@@ -1,13 +1,15 @@
 import fs from 'fs';
 import path from 'path';
 import { pathToFileURL } from 'url';
-import { loadStories as browserStoriesProvider } from './storybook/providers/browser.js';
-import { Config, Browser, BrowserConfig, Options, isDefined } from '../types.js';
+import { loadStories as browserStoriesProvider } from './providers/browser.js';
+import { Config, BrowserConfig, BrowserConfigObject, Options, isDefined } from '../types.js';
 import { configExt, loadThroughTSX } from './utils.js';
+import { CreeveyReporter, TeamcityReporter } from './reporter.js';
+import { SeleniumWebdriver } from './selenium/webdriver.js';
 
 export const defaultBrowser = 'chrome';
 
-export const defaultConfig: Omit<Config, 'gridUrl' | 'storiesProvider' | 'testsDir' | 'tsConfig'> = {
+export const defaultConfig: Omit<Config, 'gridUrl' | 'testsDir' | 'tsConfig'> = {
   disableTelemetry: false,
   useDocker: true,
   dockerImage: 'aerokube/selenoid:latest-release',
@@ -17,15 +19,19 @@ export const defaultConfig: Omit<Config, 'gridUrl' | 'storiesProvider' | 'testsD
   storybookUrl: 'http://localhost:6006',
   screenDir: path.resolve('images'),
   reportDir: path.resolve('report'),
+  reporter: process.env.TEAMCITY_VERSION ? TeamcityReporter : CreeveyReporter,
+  storiesProvider: browserStoriesProvider,
+  webdriver: SeleniumWebdriver,
   maxRetries: 0,
-  diffOptions: { threshold: 0, includeAA: true },
+  testTimeout: 30000,
+  diffOptions: { threshold: 0.05, includeAA: false },
+  odiffOptions: { threshold: 0.05, antialiasing: true },
   browsers: { [defaultBrowser]: true },
   hooks: {},
-  babelOptions: (_) => _,
   testsRegex: /\.creevey\.(t|j)s$/,
 };
 
-function normalizeBrowserConfig(name: string, config: Browser): BrowserConfig {
+function normalizeBrowserConfig(name: string, config: BrowserConfig): BrowserConfigObject {
   if (typeof config == 'boolean') return { browserName: name };
   if (typeof config == 'string') return { browserName: config };
   return config;
@@ -65,8 +71,6 @@ export async function readConfig(options: Options): Promise<Config> {
     Object.assign(userConfig, configData);
   }
 
-  if (!userConfig.storiesProvider) userConfig.storiesProvider = browserStoriesProvider;
-
   if (options.failFast != undefined) userConfig.failFast = Boolean(options.failFast);
   if (options.reportDir) userConfig.reportDir = path.resolve(options.reportDir);
   if (options.screenDir) userConfig.screenDir = path.resolve(options.screenDir);

package/src/server/docker.ts

@@ -1,16 +1,14 @@
-import cluster from 'cluster';
-import ora from 'ora';
+import tar from 'tar-stream';
 import { Writable } from 'stream';
 import Dockerode, { Container } from 'dockerode';
-import { Config, BrowserConfig, isDockerMessage, DockerAuth } from '../types.js';
-import { subscribeOn, sendDockerMessage, emitDockerMessage } from './messages.js';
-import { isInsideDocker, LOCALHOST_REGEXP } from './utils.js';
+import { DockerAuth } from '../types.js';
+import { subscribeOn } from './messages.js';
 import { logger } from './logger.js';
 
 const docker = new Dockerode();
 
 class DevNull extends Writable {
-  _write(_chunk: unknown, _encoding: BufferEncoding, callback: (error?: Error | null | undefined) => void): void {
+  _write(_chunk: unknown, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
     setImmediate(callback);
   }
 }
@@ -24,13 +22,15 @@ export async function pullImages(
   if (platform) args.platform = platform;
 
   logger.info('Pull docker images');
+  // TODO Replace with `import from`
+  const { default: yoctoSpinner } = await import('yocto-spinner');
   for (const image of images) {
     await new Promise<void>((resolve, reject) => {
-      const spinner = ora(`${image}: Pull start`).start();
+      const spinner = yoctoSpinner({ text: `${image}: Pull start` }).start();
 
       docker.pull(image, args, (pullError: Error | null, stream?: NodeJS.ReadableStream) => {
         if (pullError || !stream) {
-          spinner.fail();
+          spinner.error(pullError?.message);
           reject(pullError ?? new Error('Unknown error'));
           return;
         }
@@ -39,11 +39,11 @@ export async function pullImages(
 
         function onFinished(error: Error | null): void {
           if (error) {
-            spinner.fail();
+            spinner.error(error.message);
             reject(error);
             return;
           }
-          spinner.succeed(`${image}: Pull complete`);
+          spinner.success(`${image}: Pull complete`);
           resolve();
         }
 
@@ -57,6 +57,54 @@ export async function pullImages(
   }
 }
 
+export async function buildImage(imageName: string, dockerfile: string): Promise<void> {
+  const images = await docker.listImages({ filters: { label: [`creevey=${imageName}`] } });
+
+  if (images.at(0)) {
+    logger.info(`Image ${imageName} already exists`);
+    return;
+  }
+
+  const pack = tar.pack();
+  pack.entry({ name: 'Dockerfile' }, dockerfile);
+  pack.finalize();
+
+  const { default: yoctoSpinner } = await import('yocto-spinner');
+  const spinner = yoctoSpinner({ text: `${imageName}: Build start` }).start();
+  await new Promise<void>((resolve, reject) => {
+    void docker.buildImage(
+      // @ts-expect-error Type incompatibility AsyncIterator and AsyncIterableIterator
+      pack,
+      { t: imageName, labels: { creevey: imageName } },
+      (buildError: Error | null, stream) => {
+        if (buildError || !stream) {
+          spinner.error(buildError?.message);
+          reject(buildError ?? new Error('Unknown error'));
+          return;
+        }
+
+        docker.modem.followProgress(stream, onFinished, onProgress);
+
+        function onFinished(error: Error | null): void {
+          if (error) {
+            spinner.error(error.message);
+            reject(error);
+            return;
+          }
+          spinner.success(`${imageName}: Build complete`);
+          resolve();
+        }
+
+        function onProgress(event: { id: string; status: string; progress?: string }): void {
+          if (!/^[a-z0-9]{12}$/i.test(event.id)) return;
+
+          spinner.text = `${imageName}: [${event.id}] ${event.status} ${event.progress ? event.progress : ''}`;
+        }
+      },
+    );
+  });
+}
+
 export async function runImage(
   image: string,
   args: string[],
@@ -100,37 +148,3 @@ export async function runImage(
     );
   });
 }
-
-export async function initDocker(
-  config: Config,
-  browser: string | undefined,
-  startContainer: () => Promise<string>,
-): Promise<void> {
-  if (cluster.isPrimary) {
-    const host = await startContainer();
-    let gridUrl = 'http://localhost:4444/wd/hub';
-    gridUrl = isInsideDocker ? gridUrl.replace(LOCALHOST_REGEXP, host) : gridUrl;
-    cluster.on('message', (worker, message: unknown) => {
-      if (!isDockerMessage(message)) return;
-
-      const dockerMessage = message;
-      if (dockerMessage.type != 'start') return;
-
-      sendDockerMessage(worker, {
-        type: 'success',
-        payload: { gridUrl },
-      });
-    });
-  } else {
-    if (browser && (config.browsers[browser] as BrowserConfig).gridUrl) return Promise.resolve();
-    return new Promise((resolve) => {
-      subscribeOn('docker', (message) => {
-        if (message.type == 'success') {
-          config.gridUrl = message.payload.gridUrl;
-          resolve();
-        }
-      });
-      emitDockerMessage({ type: 'start' });
-    });
-  }
-}

package/src/server/index.ts

@@ -1,30 +1,70 @@
 import cluster from 'cluster';
 import { readConfig, defaultBrowser } from './config.js';
-import { Options, Config, BrowserConfig } from '../types.js';
+import { Options, Config, BrowserConfigObject, isWorkerMessage } from '../types.js';
 import { logger } from './logger.js';
+import { SeleniumWebdriver } from './selenium/webdriver.js';
+import { LOCALHOST_REGEXP } from './webdriver.js';
+import { isInsideDocker } from './utils.js';
+import { sendWorkerMessage } from './messages.js';
 
-// NOTE: Impure function, mutate config by adding gridUrl prop
-async function startWebdriverServer(config: Config, options: Options): Promise<void> {
-  if (config.useDocker) {
-    return (await import('./docker.js')).initDocker(config, options.browser, async () =>
-      (await import('./selenium/selenoid.js')).startSelenoidContainer(config, options.debug),
-    );
+async function startWebdriverServer(browser: string, config: Config, options: Options): Promise<string | undefined> {
+  if (config.webdriver === SeleniumWebdriver) {
+    if (cluster.isPrimary) {
+      // TODO Get random free port
+      const { startSelenoidContainer, startSelenoidStandalone } = await import('./selenium/selenoid.js');
+      const gridUrl = 'http://localhost:4444/wd/hub';
+      if (config.useDocker) {
+        const host = await startSelenoidContainer(config, options.debug);
+        return isInsideDocker ? gridUrl.replace(LOCALHOST_REGEXP, host) : gridUrl;
+      }
+      await startSelenoidStandalone(config, options.debug);
+      return gridUrl;
+    }
+    // TODO Worker might want to use docker image of browser or start standalone selenium
   } else {
-    return (await import('./selenium/selenoid.js')).startSelenoidStandalone(config, options.debug);
+    // TODO start standalone playwright server (useDocker == false)
+
+    if (cluster.isWorker) {
+      // TODO Re-use dockerImage
+
+      const { startPlaywrightContainer } = await import('./playwright/docker.js');
+      const { browserName } = config.browsers[browser] as BrowserConfigObject;
+      const host = await startPlaywrightContainer(browserName, options.debug);
+
+      return host;
+    } else {
+      const { default: getPort } = await import('get-port');
+
+      cluster.on('message', (worker, message: unknown) => {
+        if (!isWorkerMessage(message)) return;
+
+        const workerMessage = message;
+        if (workerMessage.type != 'port') return;
+
+        void getPort().then((port) => {
+          sendWorkerMessage(worker, {
+            type: 'port',
+            payload: { port },
+          });
+        });
+      });
+    }
+    // TODO Support gridUrl for playwright
+    // NOTE: There is no grid for playwright right now
   }
 }
 
 export default async function (options: Options): Promise<void> {
   const config = await readConfig(options);
-  const { browser = defaultBrowser, tests, update, ui, port } = options;
+  const { browser = defaultBrowser, update, ui, port } = options;
+  let gridUrl = cluster.isPrimary ? config.gridUrl : options.gridUrl;
 
-  // NOTE: We don't need docker nor selenoid for webpack or update options
+  // NOTE: We don't need docker nor selenoid for update option
   if (
-    !(config.gridUrl || (Object.values(config.browsers) as BrowserConfig[]).every(({ gridUrl }) => gridUrl)) &&
-    !tests &&
+    !(gridUrl || (Object.values(config.browsers) as BrowserConfigObject[]).every(({ gridUrl }) => gridUrl)) &&
     !update
   ) {
-    await startWebdriverServer(config, options);
+    gridUrl = await startWebdriverServer(browser, config, options);
   }
 
   switch (true) {
@@ -37,15 +77,14 @@ export default async function (options: Options): Promise<void> {
 
       const resolveApi = (await import('./master/server.js')).start(config.reportDir, port, ui);
 
-      return (await import('./master/index.js')).start(config, options, resolveApi);
+      return (await import('./master/start.js')).start(gridUrl, config, options, resolveApi);
     }
     default: {
       logger.info(`Starting Worker for ${browser}`);
 
-      return (await import('./worker/index.js')).start(config, {
-        ...options,
-        browser,
-      });
+      // NOTE: We assume that we pass `gridUrl` to worker CLI options
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      return (await import('./worker/start.js')).start(browser, gridUrl!, config, options);
     }
   }
 }

package/src/server/master/master.ts

@@ -21,11 +21,8 @@ function mergeTests(
   return testsFromStories;
 }
 
-export default async function master(
-  config: Config,
-  options: { watch: boolean; debug: boolean; port: number },
-): Promise<Runner> {
-  const runner = new Runner(config);
+export default async function master(config: Config, gridUrl?: string): Promise<Runner> {
+  const runner = new Runner(config, gridUrl);
   const reportDataPath = path.join(config.reportDir, 'data.js');
   const testsFromReport = tryToLoadTestsData(reportDataPath) ?? {};
 
@@ -33,7 +30,7 @@ export default async function master(
 
   const tests = await loadTestsFromStories(
     Object.keys(config.browsers),
-    (listener) => config.storiesProvider(config, options, listener),
+    (listener) => config.storiesProvider(config, listener),
     (testsDiff) => {
       runner.updateTests(testsDiff);
       saveTestsJson(runner.tests, config.reportDir);

package/src/server/master/pool.ts

@@ -1,6 +1,14 @@
 import cluster, { Worker as ClusterWorker } from 'cluster';
 import { EventEmitter } from 'events';
-import { Worker, Config, TestResult, BrowserConfig, WorkerMessage, TestStatus, isWorkerMessage } from '../../types.js';
+import {
+  Worker,
+  Config,
+  TestResult,
+  BrowserConfigObject,
+  WorkerMessage,
+  TestStatus,
+  isWorkerMessage,
+} from '../../types.js';
 import { sendTestMessage, sendShutdownMessage, subscribeOnWorker } from '../messages.js';
 import { isShuttingDown } from '../utils.js';
 
@@ -14,23 +22,26 @@ interface WorkerTest {
 
 export default class Pool extends EventEmitter {
   private maxRetries: number;
-  private config: BrowserConfig;
+  private config: BrowserConfigObject;
   private workers: Worker[] = [];
   private queue: WorkerTest[] = [];
   private forcedStop = false;
   private failFast: boolean;
+  private gridUrl?: string;
   public get isRunning(): boolean {
     return this.workers.length !== this.freeWorkers.length;
   }
   constructor(
     config: Config,
     private browser: string,
+    gridUrl?: string,
   ) {
     super();
 
     this.failFast = config.failFast;
     this.maxRetries = config.maxRetries;
-    this.config = config.browsers[browser] as BrowserConfig;
+    this.config = config.browsers[browser] as BrowserConfigObject;
+    this.gridUrl = this.config.gridUrl ?? gridUrl;
   }
 
   async init(): Promise<void> {
@@ -96,11 +107,13 @@ export default class Pool extends EventEmitter {
   }
 
   private getFreeWorker(): Worker | undefined {
-    return this.freeWorkers[Math.floor(Math.random() * this.freeWorkers.length)];
+    const freeWorkers = this.freeWorkers;
+
+    return freeWorkers[Math.floor(Math.random() * freeWorkers.length)];
   }
 
   private get aliveWorkers(): Worker[] {
-    return this.workers.filter((worker) => !worker.exitedAfterDisconnect);
+    return this.workers.filter((worker) => !worker.exitedAfterDisconnect && !worker.isShuttingDown);
   }
 
   private get freeWorkers(): Worker[] {
@@ -109,12 +122,12 @@ export default class Pool extends EventEmitter {
 
   private async forkWorker(retry = 0): Promise<Worker | { error: string }> {
     cluster.setupPrimary({
-      args: ['--browser', this.browser, ...process.argv.slice(2)],
+      args: ['--browser', this.browser, ...(this.gridUrl ? ['--gridUrl', this.gridUrl] : []), ...process.argv.slice(2)],
     });
     const worker = cluster.fork();
     const message = await new Promise((resolve: (value: WorkerMessage) => void) => {
       const readyHandler = (message: unknown): void => {
-        if (!isWorkerMessage(message)) return;
+        if (!isWorkerMessage(message) || message.type == 'port') return;
         worker.off('message', readyHandler);
         resolve(message);
       };
@@ -146,6 +159,7 @@ export default class Pool extends EventEmitter {
   }
 
   private gracefullyKill(worker: Worker): void {
+    worker.isShuttingDown = true;
     const timeout = setTimeout(() => {
       worker.kill();
     }, 10000);
@@ -185,9 +199,11 @@ export default class Pool extends EventEmitter {
           unsubscribe();
         });
 
-        this.gracefullyKill(worker);
+        if (message.payload.subtype == 'unknown') {
+          this.gracefullyKill(worker);
+        }
 
-        this.handleTestResult(worker, test, { status: 'failed', ...message.payload });
+        this.handleTestResult(worker, test, { status: 'failed', error: message.payload.error });
       }),
       subscribeOnWorker(worker, 'test', (message) => {
         if (message.type != 'end') return;

package/src/server/master/runner.ts

@@ -24,7 +24,7 @@ export default class Runner extends EventEmitter {
   public get isRunning(): boolean {
     return Object.values(this.pools).some((pool) => pool.isRunning);
   }
-  constructor(config: Config) {
+  constructor(config: Config, gridUrl?: string) {
     super();
 
     this.failFast = config.failFast;
@@ -32,7 +32,7 @@ export default class Runner extends EventEmitter {
     this.reportDir = config.reportDir;
     this.browsers = Object.keys(config.browsers);
     this.browsers
-      .map((browser) => (this.pools[browser] = new Pool(config, browser)))
+      .map((browser) => (this.pools[browser] = new Pool(config, browser, gridUrl)))
       .map((pool) => pool.on('test', this.handlePoolMessage));
   }
 
@@ -202,12 +202,14 @@ export default class Runner extends EventEmitter {
           test.approved = {};
         }
         test.approved[name] = retry;
+        test.status = 'approved';
 
         updatedTests[test.id] = {
           id: test.id,
           browser: test.browser,
           storyPath: test.storyPath,
           storyId: test.storyId,
+          status: test.status,
           approved: { [name]: retry },
         };
       }

package/src/server/master/server.ts

@@ -21,6 +21,7 @@ export function start(reportDir: string, port: number, ui: boolean): (api: Creev
   let setStoriesCounter = 0;
   const creeveyApi = new Promise<CreeveyApi>((resolve) => (resolveApi = resolve));
   const app = new Koa();
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
   const server = http.createServer(app.callback());
   const wss = new WebSocket.Server({ server });
 

package/src/server/master/{index.ts → start.ts}

@@ -5,7 +5,7 @@ import { copyFile, readdir, mkdir, writeFile } from 'fs/promises';
 import master from './master.js';
 import creeveyApi, { CreeveyApi } from './api.js';
 import { Config, Options, TestData, isDefined } from '../../types.js';
-import { shutdown, shutdownWorkers, testsToImages, readDirRecursive } from '../utils.js';
+import { shutdownWorkers, testsToImages, readDirRecursive } from '../utils.js';
 import { subscribeOn } from '../messages.js';
 import Runner from './runner.js';
 import { logger } from '../logger.js';
@@ -46,13 +46,17 @@ function outputUnnecessaryImages(imagesDir: string, images: Set<string>): void {
   }
 }
 
-export async function start(config: Config, options: Options, resolveApi: (api: CreeveyApi) => void): Promise<void> {
+export async function start(
+  gridUrl: string | undefined,
+  config: Config,
+  options: Options,
+  resolveApi: (api: CreeveyApi) => void,
+): Promise<void> {
   let runner: Runner | null = null;
   if (config.hooks.before) {
     await config.hooks.before();
   }
   subscribeOn('shutdown', () => config.hooks.after?.());
-  process.removeListener('SIGINT', shutdown);
   process.on('SIGINT', () => {
     runner?.removeAllListeners('stop');
     if (runner?.isRunning) {
@@ -67,15 +71,13 @@ export async function start(config: Config, options: Options, resolveApi: (api:
     }
   });
 
-  runner = await master(config, { watch: options.ui, debug: options.debug, port: options.port });
+  runner = await master(config, gridUrl);
 
-  if (options.saveReport) {
-    runner.on('stop', () => {
-      void copyStatics(config.reportDir).then(() =>
-        writeFile(path.join(config.reportDir, 'data.js'), reportDataModule(runner.status.tests)),
-      );
-    });
-  }
+  runner.on('stop', () => {
+    void copyStatics(config.reportDir).then(() =>
+      writeFile(path.join(config.reportDir, 'data.js'), reportDataModule(runner.status.tests)),
+    );
+  });
 
   if (options.ui) {
     resolveApi(creeveyApi(runner));