creevey 0.9.0-beta.1 → 0.9.0-beta.3

package/lib/esm/server/master/index.js

@@ -0,0 +1,125 @@
+import path from 'path';
+import { writeFileSync, copyFile, readdir, mkdir, existsSync } from 'fs';
+import { promisify } from 'util';
+import master from './master';
+import creeveyApi from './api';
+import { isDefined } from '../../types';
+import { shutdown, shutdownWorkers, testsToImages, readDirRecursive } from '../utils';
+import { subscribeOn } from '../messages';
+import { logger } from '../logger';
+const copyFileAsync = promisify(copyFile);
+const readdirAsync = promisify(readdir);
+const mkdirAsync = promisify(mkdir);
+
+async function copyStatics(reportDir) {
+  const clientDir = path.join(__dirname, '../../client/web');
+  const files = (await readdirAsync(clientDir, {
+    withFileTypes: true
+  })).filter(dirent => dirent.isFile() && !dirent.name.endsWith('.d.ts') && !dirent.name.endsWith('.tsx')).map(dirent => dirent.name);
+  await mkdirAsync(reportDir, {
+    recursive: true
+  });
+
+  for (const file of files) {
+    await copyFileAsync(path.join(clientDir, file), path.join(reportDir, file));
+  }
+}
+
+function reportDataModule(data) {
+  return `
+(function (root, factory) {
+  if (typeof module === 'object' && module.exports) {
+    module.exports = factory();
+  } else {
+    root.__CREEVEY_DATA__ = factory();
+  }
+}(this, function () { return ${JSON.stringify(data)} }));
+`;
+}
+
+function outputUnnecessaryImages(imagesDir, images) {
+  if (!existsSync(imagesDir)) return;
+  const unnecessaryImages = readDirRecursive(imagesDir).map(imagePath => path.posix.relative(imagesDir, imagePath)).filter(imagePath => !images.has(imagePath));
+
+  if (unnecessaryImages.length > 0) {
+    logger.warn('We found unnecessary screenshot images, those can be safely removed:\n', unnecessaryImages.join('\n'));
+  }
+}
+
+export default async function (config, options, resolveApi) {
+  let runner = null;
+
+  if (config.hooks.before) {
+    await config.hooks.before();
+  }
+
+  subscribeOn('shutdown', () => {
+    var _config$hooks$after, _config$hooks;
+
+    return (_config$hooks$after = (_config$hooks = config.hooks).after) === null || _config$hooks$after === void 0 ? void 0 : _config$hooks$after.call(_config$hooks);
+  });
+  process.removeListener('SIGINT', shutdown);
+  process.on('SIGINT', () => {
+    var _runner, _runner2;
+
+    (_runner = runner) === null || _runner === void 0 ? void 0 : _runner.removeAllListeners('stop');
+
+    if ((_runner2 = runner) !== null && _runner2 !== void 0 && _runner2.isRunning) {
+      var _runner4;
+
+      // TODO Better handle stop
+      void Promise.race([new Promise(resolve => setTimeout(resolve, 10000)), new Promise(resolve => {
+        var _runner3;
+
+        return (_runner3 = runner) === null || _runner3 === void 0 ? void 0 : _runner3.once('stop', resolve);
+      })]).then(() => shutdownWorkers());
+      (_runner4 = runner) === null || _runner4 === void 0 ? void 0 : _runner4.stop();
+    } else {
+      void shutdownWorkers();
+    }
+  });
+  runner = await master(config, {
+    watch: options.ui,
+    debug: options.debug,
+    port: options.port
+  });
+
+  if (options.saveReport) {
+    await copyStatics(config.reportDir);
+    runner.on('stop', () => {
+      var _runner5;
+
+      return writeFileSync(path.join(config.reportDir, 'data.js'), reportDataModule((_runner5 = runner) === null || _runner5 === void 0 ? void 0 : _runner5.status.tests));
+    });
+  }
+
+  if (options.ui) {
+    resolveApi(creeveyApi(runner));
+    logger.info(`Started on http://localhost:${options.port}`);
+  } else {
+    if (Object.values(runner.status.tests).filter(test => test && !test.skip).length == 0) {
+      logger.warn("Don't have any tests to run"); // eslint-disable-next-line no-process-exit
+
+      void shutdownWorkers().then(() => process.exit());
+      return;
+    }
+
+    runner.once('stop', () => {
+      var _runner6;
+
+      const tests = Object.values(((_runner6 = runner) === null || _runner6 === void 0 ? void 0 : _runner6.status.tests) ?? {});
+      const isSuccess = tests.filter(isDefined).filter(({
+        skip
+      }) => !skip).every(({
+        status
+      }) => status == 'success'); // TODO output summary
+
+      process.exitCode = isSuccess ? 0 : -1;
+      if (!config.failFast) outputUnnecessaryImages(config.screenDir, testsToImages(tests)); // eslint-disable-next-line no-process-exit
+
+      void shutdownWorkers().then(() => process.exit());
+    }); // TODO grep
+
+    runner.start(Object.keys(runner.status.tests));
+  }
+}

package/lib/esm/server/master/master.js

@@ -0,0 +1,38 @@
+import path from 'path';
+import { isDefined } from '../../types';
+import { loadTestsFromStories, saveTestsJson } from '../stories';
+import Runner from './runner';
+import { startWebpackCompiler } from '../loaders/webpack/start';
+
+function mergeTests(testsWithReports, testsFromStories) {
+  Object.values(testsFromStories).filter(isDefined).forEach(test => {
+    const testWithReport = testsWithReports[test.id];
+    if (!testWithReport) return;
+    test.retries = testWithReport.retries;
+    if (testWithReport.status == 'success' || testWithReport.status == 'failed') test.status = testWithReport.status;
+    test.results = testWithReport.results;
+    test.approved = testWithReport.approved;
+  });
+  return testsFromStories;
+}
+
+export default async function master(config, options) {
+  if (config.useWebpackToExtractTests) await startWebpackCompiler();
+  const runner = new Runner(config);
+  const reportDataPath = path.join(config.reportDir, 'data.js');
+  let testsFromReport = {};
+
+  try {
+    testsFromReport = await import(reportDataPath);
+  } catch (error) {// Ignore error
+  }
+
+  await runner.init();
+  const tests = await loadTestsFromStories(Object.keys(config.browsers), listener => config.storiesProvider(config, options, listener), testsDiff => {
+    runner.updateTests(testsDiff);
+    saveTestsJson(runner.tests, config.reportDir);
+  });
+  runner.tests = mergeTests(testsFromReport, tests);
+  saveTestsJson(runner.tests, config.reportDir);
+  return runner;
+}

package/lib/esm/server/master/pool.js

@@ -0,0 +1,176 @@
+import cluster, { Worker as ClusterWorker } from 'cluster';
+import { EventEmitter } from 'events';
+import { isWorkerMessage } from '../../types';
+import { sendTestMessage, sendShutdownMessage, subscribeOnWorker } from '../messages';
+import { isShuttingDown } from '../utils';
+const FORK_RETRIES = 5;
+export default class Pool extends EventEmitter {
+  workers = [];
+  queue = [];
+  forcedStop = false;
+
+  get isRunning() {
+    return this.workers.length !== this.freeWorkers.length;
+  }
+
+  constructor(config, browser) {
+    super();
+    this.browser = browser;
+    this.failFast = config.failFast;
+    this.maxRetries = config.maxRetries;
+    this.config = config.browsers[browser];
+  }
+
+  async init() {
+    const poolSize = this.config.limit || 1;
+    this.workers = (await Promise.all(Array.from({
+      length: poolSize
+    }).map(() => this.forkWorker()))).filter(workerOrError => workerOrError instanceof ClusterWorker);
+    if (this.workers.length != poolSize) throw new Error(`Can't instantiate workers for ${this.browser} due many errors`);
+    this.workers.forEach(worker => this.exitHandler(worker));
+  }
+
+  start(tests) {
+    if (this.isRunning) return false;
+    this.queue = tests.map(({
+      id,
+      path
+    }) => ({
+      id,
+      path,
+      retries: 0
+    }));
+    this.process();
+    return true;
+  }
+
+  stop() {
+    if (!this.isRunning) {
+      this.emit('stop');
+      return;
+    }
+
+    this.forcedStop = true;
+    this.queue = [];
+  }
+
+  process() {
+    const worker = this.getFreeWorker();
+    const [test] = this.queue;
+
+    if (this.queue.length == 0 && this.workers.length === this.freeWorkers.length) {
+      this.forcedStop = false;
+      this.emit('stop');
+      return;
+    }
+
+    if (!worker || !test) return;
+    worker.isRunning = true;
+    const {
+      id
+    } = test;
+    this.queue.shift();
+    this.sendStatus({
+      id,
+      status: 'running'
+    });
+    this.subscribe(worker, test);
+    sendTestMessage(worker, {
+      type: 'start',
+      payload: test
+    });
+    this.process();
+  }
+
+  sendStatus(message) {
+    this.emit('test', message);
+  }
+
+  getFreeWorker() {
+    return this.freeWorkers[Math.floor(Math.random() * this.freeWorkers.length)];
+  }
+
+  get aliveWorkers() {
+    return this.workers.filter(worker => !worker.exitedAfterDisconnect);
+  }
+
+  get freeWorkers() {
+    return this.aliveWorkers.filter(worker => !worker.isRunning);
+  }
+
+  async forkWorker(retry = 0) {
+    cluster.setupMaster({
+      args: ['--browser', this.browser, ...process.argv.slice(2)]
+    });
+    const worker = cluster.fork();
+    const message = await new Promise(resolve => {
+      const readyHandler = message => {
+        if (!isWorkerMessage(message)) return;
+        worker.off('message', readyHandler);
+        resolve(message);
+      };
+
+      worker.on('message', readyHandler);
+    });
+    if (message.type != 'error') return worker;
+    this.gracefullyKill(worker);
+    if (retry == FORK_RETRIES) return message.payload;
+    return this.forkWorker(retry + 1);
+  }
+
+  exitHandler(worker) {
+    // eslint-disable-next-line @typescript-eslint/no-misused-promises
+    worker.once('exit', async () => {
+      if (isShuttingDown.current) return;
+      const workerOrError = await this.forkWorker();
+      if (!(workerOrError instanceof ClusterWorker)) throw new Error(`Can't instantiate worker for ${this.browser} due many errors`);
+      this.exitHandler(workerOrError);
+      this.workers[this.workers.indexOf(worker)] = workerOrError;
+      this.process();
+    });
+  }
+
+  gracefullyKill(worker) {
+    const timeout = setTimeout(() => worker.kill(), 10000);
+    worker.on('exit', () => clearTimeout(timeout));
+    sendShutdownMessage(worker);
+  }
+
+  shouldRetry(test) {
+    return test.retries < this.maxRetries && !this.forcedStop;
+  }
+
+  handleTestResult(worker, test, result) {
+    const shouldRetry = result.status == 'failed' && this.shouldRetry(test);
+
+    if (shouldRetry) {
+      test.retries += 1;
+      this.queue[this.failFast ? 'unshift' : 'push'](test);
+    }
+
+    this.sendStatus({
+      id: test.id,
+      status: shouldRetry ? 'retrying' : result.status,
+      result
+    });
+    worker.isRunning = false;
+    setImmediate(() => this.process());
+  }
+
+  subscribe(worker, test) {
+    const subscriptions = [subscribeOnWorker(worker, 'worker', message => {
+      if (message.type != 'error') return;
+      subscriptions.forEach(unsubscribe => unsubscribe());
+      this.gracefullyKill(worker);
+      this.handleTestResult(worker, test, {
+        status: 'failed',
+        ...message.payload
+      });
+    }), subscribeOnWorker(worker, 'test', message => {
+      if (message.type != 'end') return;
+      subscriptions.forEach(unsubscribe => unsubscribe());
+      this.handleTestResult(worker, test, message.payload);
+    })];
+  }
+
+}

package/lib/esm/server/master/runner.js

@@ -0,0 +1,259 @@
+import path from 'path';
+import { copyFile, mkdir } from 'fs';
+import { promisify } from 'util';
+import { EventEmitter } from 'events';
+import { isDefined } from '../../types';
+import Pool from './pool';
+const copyFileAsync = promisify(copyFile);
+const mkdirAsync = promisify(mkdir);
+export default class Runner extends EventEmitter {
+  pools = {};
+  tests = {};
+
+  get isRunning() {
+    return Object.values(this.pools).some(pool => pool.isRunning);
+  }
+
+  constructor(config) {
+    super();
+    this.failFast = config.failFast;
+    this.screenDir = config.screenDir;
+    this.reportDir = config.reportDir;
+    this.browsers = Object.keys(config.browsers);
+    this.browsers.map(browser => this.pools[browser] = new Pool(config, browser)).map(pool => pool.on('test', this.handlePoolMessage));
+  }
+
+  handlePoolMessage = message => {
+    const {
+      id,
+      status,
+      result
+    } = message;
+    const test = this.tests[id];
+    if (!test) return;
+    const {
+      browser,
+      testName,
+      storyPath,
+      storyId
+    } = test; // TODO Handle 'retrying' status
+
+    test.status = status == 'retrying' ? 'failed' : status;
+
+    if (!result) {
+      this.sendUpdate({
+        tests: {
+          [id]: {
+            id,
+            browser,
+            testName,
+            storyPath,
+            status: test.status,
+            storyId
+          }
+        }
+      });
+      return;
+    }
+
+    if (!test.results) {
+      test.results = [];
+    }
+
+    test.results.push(result);
+    this.sendUpdate({
+      tests: {
+        [id]: {
+          id,
+          browser,
+          testName,
+          storyPath,
+          status: test.status,
+          results: [result],
+          storyId
+        }
+      }
+    });
+    if (this.failFast && status == 'failed') this.stop();
+  };
+  handlePoolStop = () => {
+    if (!this.isRunning) {
+      this.sendUpdate({
+        isRunning: false
+      });
+      this.emit('stop');
+    }
+  };
+
+  async init() {
+    await Promise.all(Object.values(this.pools).map(pool => pool.init()));
+  }
+
+  updateTests(testsDiff) {
+    const tests = {};
+    const removedTests = [];
+    Object.entries(testsDiff).forEach(([id, newTest]) => {
+      const oldTest = this.tests[id];
+
+      if (newTest) {
+        if (oldTest) {
+          this.tests[id] = { ...newTest,
+            retries: oldTest.retries,
+            results: oldTest.results,
+            approved: oldTest.approved
+          };
+        } else this.tests[id] = newTest; // eslint-disable-next-line @typescript-eslint/no-unused-vars
+
+
+        const {
+          story,
+          fn,
+          ...restTest
+        } = newTest;
+        tests[id] = { ...restTest,
+          status: 'unknown'
+        };
+      } else if (oldTest) {
+        const {
+          id,
+          browser,
+          testName,
+          storyPath,
+          storyId
+        } = oldTest;
+        removedTests.push({
+          id,
+          browser,
+          testName,
+          storyPath,
+          storyId
+        });
+        delete this.tests[id];
+      }
+    });
+    this.sendUpdate({
+      tests,
+      removedTests
+    });
+  }
+
+  start(ids) {
+    if (this.isRunning) return;
+    const testsToStart = ids.map(id => this.tests[id]).filter(isDefined).filter(test => !test.skip);
+    if (testsToStart.length == 0) return;
+    this.sendUpdate({
+      isRunning: true,
+      tests: testsToStart.reduce((update, {
+        id,
+        storyId,
+        browser,
+        testName,
+        storyPath
+      }) => ({ ...update,
+        [id]: {
+          id,
+          browser,
+          testName,
+          storyPath,
+          status: 'pending',
+          storyId
+        }
+      }), {})
+    });
+    const testsByBrowser = testsToStart.reduce((tests, test) => {
+      const {
+        id,
+        browser,
+        testName,
+        storyPath
+      } = test;
+      const restPath = [...storyPath, testName].filter(isDefined);
+      test.status = 'pending';
+      return { ...tests,
+        [browser]: [...(tests[browser] || []), {
+          id,
+          path: restPath
+        }]
+      };
+    }, {});
+    this.browsers.forEach(browser => {
+      const pool = this.pools[browser];
+      const tests = testsByBrowser[browser];
+
+      if (tests && tests.length > 0 && pool.start(tests)) {
+        pool.once('stop', this.handlePoolStop);
+      }
+    });
+  }
+
+  stop() {
+    if (!this.isRunning) return;
+    this.browsers.forEach(browser => this.pools[browser].stop());
+  }
+
+  get status() {
+    const tests = {};
+    Object.values(this.tests).filter(isDefined) // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    .forEach(({
+      story,
+      fn,
+      ...test
+    }) => tests[test.id] = test);
+    return {
+      isRunning: this.isRunning,
+      tests,
+      browsers: this.browsers
+    };
+  }
+
+  async approve({
+    id,
+    retry,
+    image
+  }) {
+    const test = this.tests[id];
+    if (!test || !test.results) return;
+    const result = test.results[retry];
+    if (!result || !result.images) return;
+    const images = result.images[image];
+    if (!images) return;
+
+    if (!test.approved) {
+      test.approved = {};
+    }
+
+    const {
+      browser,
+      testName,
+      storyPath
+    } = test;
+    const restPath = [...storyPath, testName].filter(isDefined);
+    const testPath = path.join(...restPath, image == browser ? '' : browser);
+    const srcImagePath = path.join(this.reportDir, testPath, images.actual);
+    const dstImagePath = path.join(this.screenDir, testPath, `${image}.png`);
+    await mkdirAsync(path.join(this.screenDir, testPath), {
+      recursive: true
+    });
+    await copyFileAsync(srcImagePath, dstImagePath);
+    test.approved[image] = retry;
+    this.sendUpdate({
+      tests: {
+        [id]: {
+          id,
+          browser,
+          testName,
+          storyPath,
+          approved: {
+            [image]: retry
+          },
+          storyId: test.storyId
+        }
+      }
+    });
+  }
+
+  sendUpdate(data) {
+    this.emit('update', data);
+  }
+
+}

package/lib/esm/server/master/server.js

@@ -0,0 +1,107 @@
+import path from 'path';
+import http from 'http';
+import cluster from 'cluster';
+import Koa from 'koa';
+import cors from '@koa/cors';
+import serve from 'koa-static';
+import mount from 'koa-mount';
+import body from 'koa-bodyparser';
+import WebSocket from 'ws';
+import { emitStoriesMessage, sendStoriesMessage, subscribeOn, subscribeOnWorker } from '../messages';
+import { isDefined, noop } from '../../types';
+import { logger } from '../logger';
+import { deserializeStory } from '../../shared';
+export default function server(reportDir, port, ui) {
+  let resolveApi = noop;
+  let setStoriesCounter = 0;
+  const creeveyApi = new Promise(resolve => resolveApi = resolve);
+  const app = new Koa();
+  const server = http.createServer(app.callback());
+  const wss = new WebSocket.Server({
+    server
+  });
+  app.use(cors());
+  app.use(body());
+  app.use(async (ctx, next) => {
+    if (ctx.method == 'GET' && ctx.path == '/ping') {
+      ctx.body = 'pong';
+      return;
+    }
+
+    await next();
+  });
+
+  if (ui) {
+    app.use(async (_, next) => {
+      await creeveyApi;
+      await next();
+    });
+  }
+
+  app.use(async (ctx, next) => {
+    if (ctx.method == 'POST' && ctx.path == '/stories') {
+      const {
+        setStoriesCounter: counter,
+        stories
+      } = ctx.request.body;
+      if (setStoriesCounter >= counter) return;
+      const deserializedStories = stories.map(([file, stories]) => [file, stories.map(deserializeStory)]);
+      setStoriesCounter = counter;
+      emitStoriesMessage({
+        type: 'update',
+        payload: deserializedStories
+      });
+      Object.values(cluster.workers ?? {}).filter(isDefined).filter(worker => worker.isConnected()).forEach(worker => sendStoriesMessage(worker, {
+        type: 'update',
+        payload: deserializedStories
+      }));
+      return;
+    }
+
+    await next();
+  });
+  app.use(async (ctx, next) => {
+    if (ctx.method == 'POST' && ctx.path == '/capture') {
+      const {
+        workerId,
+        options
+      } = ctx.request.body;
+      const worker = Object.values(cluster.workers ?? {}).filter(isDefined).find(worker => worker.process.pid == workerId); // NOTE: Hypothetical case when someone send to us capture req and we don't have a worker with browser session for it
+
+      if (!worker) return;
+      await new Promise(resolve => {
+        const unsubscribe = subscribeOnWorker(worker, 'stories', message => {
+          if (message.type != 'capture') return;
+          unsubscribe();
+          resolve();
+        });
+        sendStoriesMessage(worker, {
+          type: 'capture',
+          payload: options
+        });
+      }); // TODO Pass screenshot result to show it in inspector
+
+      ctx.body = 'Ok';
+      return;
+    }
+
+    await next();
+  });
+  app.use(serve(path.join(__dirname, '../../client/web')));
+  app.use(mount('/report', serve(reportDir)));
+  wss.on('error', error => logger.error(error));
+  server.listen(port);
+  subscribeOn('shutdown', () => {
+    server.close();
+    wss.close();
+  });
+  void creeveyApi.then(api => {
+    api.subscribe(wss);
+    wss.on('connection', ws => {
+      ws.on('message', (message, isBinary) => {
+        api.handleMessage(ws, isBinary ? message : message.toString());
+      });
+    });
+  });
+  return resolveApi;
+}