creevey 0.9.0-beta.20 → 0.9.0-beta.21

package/src/server/docker.ts

@@ -0,0 +1,136 @@
+import cluster from 'cluster';
+import ora from 'ora';
+import { Writable } from 'stream';
+import Dockerode, { Container } from 'dockerode';
+import { Config, BrowserConfig, isDockerMessage, DockerAuth } from '../types.js';
+import { subscribeOn, sendDockerMessage, emitDockerMessage } from './messages.js';
+import { isInsideDocker, LOCALHOST_REGEXP } from './utils.js';
+import { logger } from './logger.js';
+
+const docker = new Dockerode();
+
+class DevNull extends Writable {
+  _write(_chunk: unknown, _encoding: BufferEncoding, callback: (error?: Error | null | undefined) => void): void {
+    setImmediate(callback);
+  }
+}
+
+export async function pullImages(
+  images: string[],
+  { auth, platform }: { auth?: DockerAuth; platform?: string } = {},
+): Promise<void> {
+  const args: Record<string, unknown> = {};
+  if (auth) args.authconfig = auth;
+  if (platform) args.platform = platform;
+
+  logger.info('Pull docker images');
+  for (const image of images) {
+    await new Promise<void>((resolve, reject) => {
+      const spinner = ora(`${image}: Pull start`).start();
+
+      docker.pull(image, args, (pullError: Error | null, stream?: NodeJS.ReadableStream) => {
+        if (pullError || !stream) {
+          spinner.fail();
+          reject(pullError ?? new Error('Unknown error'));
+          return;
+        }
+
+        docker.modem.followProgress(stream, onFinished, onProgress);
+
+        function onFinished(error: Error | null): void {
+          if (error) {
+            spinner.fail();
+            reject(error);
+            return;
+          }
+          spinner.succeed(`${image}: Pull complete`);
+          resolve();
+        }
+
+        function onProgress(event: { id: string; status: string; progress?: string }): void {
+          if (!/^[a-z0-9]{12}$/i.test(event.id)) return;
+
+          spinner.text = `${image}: [${event.id}] ${event.status} ${event.progress ? event.progress : ''}`;
+        }
+      });
+    });
+  }
+}
+
+export async function runImage(
+  image: string,
+  args: string[],
+  options: Record<string, unknown>,
+  debug: boolean,
+): Promise<string> {
+  await Promise.all(
+    (await docker.listContainers({ all: true, filters: { ancestor: [image] } })).map(async (info) => {
+      const container = docker.getContainer(info.Id);
+      try {
+        await container.stop();
+      } catch {
+        /* noop */
+      }
+      await container.remove();
+    }),
+  );
+
+  const hub = docker.run(image, args, debug ? process.stdout : new DevNull(), options, (error) => {
+    if (error) throw error;
+  });
+
+  return new Promise((resolve) => {
+    hub.once('container', (container: Container) => {
+      // eslint-disable-next-line @typescript-eslint/no-misused-promises
+      subscribeOn('shutdown', async () => {
+        try {
+          await container.stop();
+          await container.remove();
+        } catch {
+          /* noop */
+        }
+      });
+    });
+    hub.once(
+      'start',
+      (container: Container) =>
+        void container.inspect().then((info) => {
+          resolve(info.NetworkSettings.Networks.bridge.IPAddress);
+        }),
+    );
+  });
+}
+
+export async function initDocker(
+  config: Config,
+  browser: string | undefined,
+  startContainer: () => Promise<string>,
+): Promise<void> {
+  if (cluster.isPrimary) {
+    const host = await startContainer();
+    let gridUrl = 'http://localhost:4444/wd/hub';
+    gridUrl = isInsideDocker ? gridUrl.replace(LOCALHOST_REGEXP, host) : gridUrl;
+    cluster.on('message', (worker, message: unknown) => {
+      if (!isDockerMessage(message)) return;
+
+      const dockerMessage = message;
+      if (dockerMessage.type != 'start') return;
+
+      sendDockerMessage(worker, {
+        type: 'success',
+        payload: { gridUrl },
+      });
+    });
+  } else {
+    if (browser && (config.browsers[browser] as BrowserConfig).gridUrl) return Promise.resolve();
+    return new Promise((resolve) => {
+      subscribeOn('docker', (message) => {
+        if (message.type == 'success') {
+          config.gridUrl = message.payload.gridUrl;
+          resolve();
+        }
+      });
+      emitDockerMessage({ type: 'start' });
+    });
+  }
+}

package/src/server/index.ts

@@ -0,0 +1,51 @@
+import cluster from 'cluster';
+import { readConfig, defaultBrowser } from './config.js';
+import { Options, Config, BrowserConfig } from '../types.js';
+import { logger } from './logger.js';
+
+// NOTE: Impure function, mutate config by adding gridUrl prop
+async function startWebdriverServer(config: Config, options: Options): Promise<void> {
+  if (config.useDocker) {
+    return (await import('./docker.js')).initDocker(config, options.browser, async () =>
+      (await import('./selenium/selenoid.js')).startSelenoidContainer(config, options.debug),
+    );
+  } else {
+    return (await import('./selenium/selenoid.js')).startSelenoidStandalone(config, options.debug);
+  }
+}
+
+export default async function (options: Options): Promise<void> {
+  const config = await readConfig(options);
+  const { browser = defaultBrowser, tests, update, ui, port } = options;
+
+  // NOTE: We don't need docker nor selenoid for webpack or update options
+  if (
+    !(config.gridUrl || (Object.values(config.browsers) as BrowserConfig[]).every(({ gridUrl }) => gridUrl)) &&
+    !tests &&
+    !update
+  ) {
+    await startWebdriverServer(config, options);
+  }
+
+  switch (true) {
+    case Boolean(update): {
+      (await import('./update.js')).update(config, typeof update == 'string' ? update : undefined);
+      return;
+    }
+    case cluster.isPrimary: {
+      logger.info('Starting Master Process');
+
+      const resolveApi = (await import('./master/server.js')).start(config.reportDir, port, ui);
+
+      return (await import('./master/index.js')).start(config, options, resolveApi);
+    }
+    default: {
+      logger.info(`Starting Worker for ${browser}`);
+
+      return (await import('./worker/index.js')).start(config, {
+        ...options,
+        browser,
+      });
+    }
+  }
+}

package/{lib/esm/server/logger.js → src/server/logger.ts}

@@ -1,19 +1,21 @@
 import chalk from 'chalk';
-import { default as Logger } from 'loglevel';
+import Logger from 'loglevel';
 import prefix from 'loglevel-plugin-prefix';
-export { getLogger } from 'loglevel';
+
 export const colors = {
   TRACE: chalk.magenta,
   DEBUG: chalk.cyan,
   INFO: chalk.blue,
   WARN: chalk.yellow,
-  ERROR: chalk.red
+  ERROR: chalk.red,
 };
+
 prefix.reg(Logger);
 prefix.apply(Logger, {
   format(level, name = 'Creevey') {
-    const levelColor = colors[level.toUpperCase()];
+    const levelColor = colors[level.toUpperCase() as keyof typeof colors];
     return `[${name}:${chalk.gray(process.pid)}] ${levelColor(level)} =>`;
-  }
+  },
 });
-export const logger = Logger.getLogger('Creevey');
+
+export const logger = Logger.getLogger('Creevey');

package/src/server/master/api.ts

@@ -0,0 +1,55 @@
+import WebSocket from 'ws';
+import Runner from './runner.js';
+import { Request, Response, CreeveyUpdate } from '../../types.js';
+import { logger } from '../logger.js';
+
+export interface CreeveyApi {
+  subscribe: (wss: WebSocket.Server) => void;
+  handleMessage: (ws: WebSocket, message: WebSocket.Data) => void;
+}
+
+function broadcast(wss: WebSocket.Server, message: Response): void {
+  wss.clients.forEach((ws) => {
+    if (ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify(message));
+    }
+  });
+}
+
+export default function creeveyApi(runner: Runner): CreeveyApi {
+  return {
+    subscribe(wss: WebSocket.Server) {
+      runner.on('update', (payload: CreeveyUpdate) => {
+        broadcast(wss, { type: 'update', payload });
+      });
+    },
+
+    handleMessage(ws: WebSocket, message: WebSocket.Data) {
+      if (typeof message != 'string') {
+        logger.info('unhandled message', message);
+        return;
+      }
+
+      const command = JSON.parse(message) as Request;
+
+      switch (command.type) {
+        case 'status': {
+          ws.send(JSON.stringify({ type: 'status', payload: runner.status }));
+          return;
+        }
+        case 'start': {
+          runner.start(command.payload);
+          return;
+        }
+        case 'stop': {
+          runner.stop();
+          return;
+        }
+        case 'approve': {
+          void runner.approve(command.payload);
+          return;
+        }
+      }
+    },
+  };
+}

package/src/server/master/index.ts

@@ -0,0 +1,111 @@
+import path from 'path';
+import { existsSync } from 'fs';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { copyFile, readdir, mkdir, writeFile } from 'fs/promises';
+import master from './master.js';
+import creeveyApi, { CreeveyApi } from './api.js';
+import { Config, Options, TestData, isDefined } from '../../types.js';
+import { shutdown, shutdownWorkers, testsToImages, readDirRecursive } from '../utils.js';
+import { subscribeOn } from '../messages.js';
+import Runner from './runner.js';
+import { logger } from '../logger.js';
+import { sendScreenshotsCount } from '../telemetry.js';
+
+const importMetaUrl = pathToFileURL(__filename).href;
+
+async function copyStatics(reportDir: string): Promise<void> {
+  const clientDir = path.join(path.dirname(fileURLToPath(importMetaUrl)), '../../client/web');
+  const files = (await readdir(clientDir, { withFileTypes: true }))
+    .filter((dirent) => dirent.isFile() && !dirent.name.endsWith('.d.ts') && !dirent.name.endsWith('.tsx'))
+    .map((dirent) => dirent.name);
+  await mkdir(reportDir, { recursive: true });
+  for (const file of files) {
+    await copyFile(path.join(clientDir, file), path.join(reportDir, file));
+  }
+}
+
+function reportDataModule(data: Partial<Record<string, TestData>>): string {
+  return `
+(function (root, factory) {
+  if (typeof module === 'object' && module.exports) {
+    module.exports = factory();
+  } else {
+    root.__CREEVEY_DATA__ = factory();
+  }
+}(this, function () { return ${JSON.stringify(data)} }));
+`;
+}
+
+function outputUnnecessaryImages(imagesDir: string, images: Set<string>): void {
+  if (!existsSync(imagesDir)) return;
+  const unnecessaryImages = readDirRecursive(imagesDir)
+    .map((imagePath) => path.posix.relative(imagesDir, imagePath))
+    .filter((imagePath) => !images.has(imagePath));
+  if (unnecessaryImages.length > 0) {
+    logger.warn('We found unnecessary screenshot images, those can be safely removed:\n', unnecessaryImages.join('\n'));
+  }
+}
+
+export async function start(config: Config, options: Options, resolveApi: (api: CreeveyApi) => void): Promise<void> {
+  let runner: Runner | null = null;
+  if (config.hooks.before) {
+    await config.hooks.before();
+  }
+  subscribeOn('shutdown', () => config.hooks.after?.());
+  process.removeListener('SIGINT', shutdown);
+  process.on('SIGINT', () => {
+    runner?.removeAllListeners('stop');
+    if (runner?.isRunning) {
+      // TODO Better handle stop
+      void Promise.race([
+        new Promise((resolve) => setTimeout(resolve, 10000)),
+        new Promise((resolve) => runner?.once('stop', resolve)),
+      ]).then(() => shutdownWorkers());
+      runner.stop();
+    } else {
+      void shutdownWorkers();
+    }
+  });
+
+  runner = await master(config, { watch: options.ui, debug: options.debug, port: options.port });
+
+  if (options.saveReport) {
+    runner.on('stop', () => {
+      void copyStatics(config.reportDir).then(() =>
+        writeFile(path.join(config.reportDir, 'data.js'), reportDataModule(runner.status.tests)),
+      );
+    });
+  }
+
+  if (options.ui) {
+    resolveApi(creeveyApi(runner));
+    logger.info(`Started on http://localhost:${options.port}`);
+  } else {
+    if (Object.values(runner.status.tests).filter((test) => test && !test.skip).length == 0) {
+      logger.warn("Don't have any tests to run");
+
+      void shutdownWorkers().then(() => process.exit());
+      return;
+    }
+    runner.once('stop', () => {
+      const tests = Object.values(runner.status.tests);
+      const isSuccess = tests
+        .filter(isDefined)
+        .filter(({ skip }) => !skip)
+        .every(({ status }) => status == 'success');
+      // TODO output summary
+      process.exitCode = isSuccess ? 0 : -1;
+      if (!config.failFast) outputUnnecessaryImages(config.screenDir, testsToImages(tests));
+      void sendScreenshotsCount(config, options, runner.status)
+        .catch((reason: unknown) => {
+          const error = reason instanceof Error ? (reason.stack ?? reason.message) : (reason as string);
+          logger.warn(`Can't send telemetry: ${error}`);
+        })
+        .finally(() => {
+          void shutdownWorkers().then(() => process.exit());
+        });
+    });
+    // TODO grep
+    runner.start(Object.keys(runner.status.tests));
+  }
+}

package/src/server/master/master.ts

@@ -0,0 +1,47 @@
+import path from 'path';
+import { Config, TestData, isDefined, ServerTest } from '../../types.js';
+import { loadTestsFromStories, saveTestsJson } from '../stories.js';
+import Runner from './runner.js';
+import { tryToLoadTestsData } from '../utils.js';
+
+function mergeTests(
+  testsWithReports: Partial<Record<string, TestData>>,
+  testsFromStories: Partial<Record<string, ServerTest>>,
+): Partial<Record<string, ServerTest>> {
+  Object.values(testsFromStories)
+    .filter(isDefined)
+    .forEach((test) => {
+      const testWithReport = testsWithReports[test.id];
+      if (!testWithReport) return;
+      test.retries = testWithReport.retries;
+      if (testWithReport.status == 'success' || testWithReport.status == 'failed') test.status = testWithReport.status;
+      test.results = testWithReport.results;
+      test.approved = testWithReport.approved;
+    });
+  return testsFromStories;
+}
+
+export default async function master(
+  config: Config,
+  options: { watch: boolean; debug: boolean; port: number },
+): Promise<Runner> {
+  const runner = new Runner(config);
+  const reportDataPath = path.join(config.reportDir, 'data.js');
+  const testsFromReport = tryToLoadTestsData(reportDataPath) ?? {};
+
+  await runner.init();
+
+  const tests = await loadTestsFromStories(
+    Object.keys(config.browsers),
+    (listener) => config.storiesProvider(config, options, listener),
+    (testsDiff) => {
+      runner.updateTests(testsDiff);
+      saveTestsJson(runner.tests, config.reportDir);
+    },
+  );
+
+  runner.tests = mergeTests(testsFromReport, tests);
+  saveTestsJson(runner.tests, config.reportDir);
+
+  return runner;
+}

package/src/server/master/pool.ts

@@ -0,0 +1,203 @@
+import cluster, { Worker as ClusterWorker } from 'cluster';
+import { EventEmitter } from 'events';
+import { Worker, Config, TestResult, BrowserConfig, WorkerMessage, TestStatus, isWorkerMessage } from '../../types.js';
+import { sendTestMessage, sendShutdownMessage, subscribeOnWorker } from '../messages.js';
+import { isShuttingDown } from '../utils.js';
+
+const FORK_RETRIES = 5;
+
+interface WorkerTest {
+  id: string;
+  path: string[];
+  retries: number;
+}
+
+export default class Pool extends EventEmitter {
+  private maxRetries: number;
+  private config: BrowserConfig;
+  private workers: Worker[] = [];
+  private queue: WorkerTest[] = [];
+  private forcedStop = false;
+  private failFast: boolean;
+  public get isRunning(): boolean {
+    return this.workers.length !== this.freeWorkers.length;
+  }
+  constructor(
+    config: Config,
+    private browser: string,
+  ) {
+    super();
+
+    this.failFast = config.failFast;
+    this.maxRetries = config.maxRetries;
+    this.config = config.browsers[browser] as BrowserConfig;
+  }
+
+  async init(): Promise<void> {
+    const poolSize = Math.max(1, this.config.limit ?? 1);
+    // TODO Init queue for workers to smooth browser starting load
+    this.workers = (await Promise.all(Array.from({ length: poolSize }).map(() => this.forkWorker()))).filter(
+      (workerOrError): workerOrError is Worker => workerOrError instanceof ClusterWorker,
+    );
+    if (this.workers.length != poolSize)
+      throw new Error(`Can't instantiate workers for ${this.browser} due many errors`);
+    this.workers.forEach((worker) => {
+      this.exitHandler(worker);
+    });
+  }
+
+  start(tests: { id: string; path: string[] }[]): boolean {
+    if (this.isRunning) return false;
+
+    this.queue = tests.map(({ id, path }) => ({ id, path, retries: 0 }));
+    this.process();
+
+    return true;
+  }
+
+  stop(): void {
+    if (!this.isRunning) {
+      this.emit('stop');
+      return;
+    }
+
+    this.forcedStop = true;
+    this.queue = [];
+  }
+
+  process(): void {
+    const worker = this.getFreeWorker();
+    const test = this.queue.at(0);
+
+    if (this.queue.length == 0 && this.workers.length === this.freeWorkers.length) {
+      this.forcedStop = false;
+      this.emit('stop');
+      return;
+    }
+
+    if (!worker || !test) return;
+
+    worker.isRunning = true;
+
+    const { id } = test;
+
+    this.queue.shift();
+    this.sendStatus({ id, status: 'running' });
+
+    this.subscribe(worker, test);
+
+    sendTestMessage(worker, { type: 'start', payload: test });
+
+    this.process();
+  }
+
+  private sendStatus(message: { id: string; status: TestStatus; result?: TestResult }): void {
+    this.emit('test', message);
+  }
+
+  private getFreeWorker(): Worker | undefined {
+    return this.freeWorkers[Math.floor(Math.random() * this.freeWorkers.length)];
+  }
+
+  private get aliveWorkers(): Worker[] {
+    return this.workers.filter((worker) => !worker.exitedAfterDisconnect);
+  }
+
+  private get freeWorkers(): Worker[] {
+    return this.aliveWorkers.filter((worker) => !worker.isRunning);
+  }
+
+  private async forkWorker(retry = 0): Promise<Worker | { error: string }> {
+    cluster.setupPrimary({
+      args: ['--browser', this.browser, ...process.argv.slice(2)],
+    });
+    const worker = cluster.fork();
+    const message = await new Promise((resolve: (value: WorkerMessage) => void) => {
+      const readyHandler = (message: unknown): void => {
+        if (!isWorkerMessage(message)) return;
+        worker.off('message', readyHandler);
+        resolve(message);
+      };
+      worker.on('message', readyHandler);
+    });
+
+    if (message.type != 'error') return worker;
+
+    this.gracefullyKill(worker);
+
+    if (retry == FORK_RETRIES) return message.payload;
+    return this.forkWorker(retry + 1);
+  }
+
+  private exitHandler(worker: Worker): void {
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    worker.once('exit', async () => {
+      if (isShuttingDown.current) return;
+
+      const workerOrError = await this.forkWorker();
+
+      if (!(workerOrError instanceof ClusterWorker))
+        throw new Error(`Can't instantiate worker for ${this.browser} due many errors`);
+
+      this.exitHandler(workerOrError);
+      this.workers[this.workers.indexOf(worker)] = workerOrError;
+      this.process();
+    });
+  }
+
+  private gracefullyKill(worker: Worker): void {
+    const timeout = setTimeout(() => {
+      worker.kill();
+    }, 10000);
+    worker.on('exit', () => {
+      clearTimeout(timeout);
+    });
+    sendShutdownMessage(worker);
+  }
+
+  private shouldRetry(test: WorkerTest): boolean {
+    return test.retries < this.maxRetries && !this.forcedStop;
+  }
+
+  private handleTestResult(worker: Worker, test: WorkerTest, result: TestResult): void {
+    const shouldRetry = result.status == 'failed' && this.shouldRetry(test);
+
+    if (shouldRetry) {
+      test.retries += 1;
+      this.queue[this.failFast ? 'unshift' : 'push'](test);
+    }
+
+    this.sendStatus({ id: test.id, status: shouldRetry ? 'retrying' : result.status, result });
+
+    worker.isRunning = false;
+
+    setImmediate(() => {
+      this.process();
+    });
+  }
+
+  private subscribe(worker: Worker, test: WorkerTest): void {
+    const subscriptions = [
+      subscribeOnWorker(worker, 'worker', (message) => {
+        if (message.type != 'error') return;
+
+        subscriptions.forEach((unsubscribe) => {
+          unsubscribe();
+        });
+
+        this.gracefullyKill(worker);
+
+        this.handleTestResult(worker, test, { status: 'failed', ...message.payload });
+      }),
+      subscribeOnWorker(worker, 'test', (message) => {
+        if (message.type != 'end') return;
+
+        subscriptions.forEach((unsubscribe) => {
+          unsubscribe();
+        });
+
+        this.handleTestResult(worker, test, message.payload);
+      }),
+    ];
+  }
+}