creevey 0.10.0-beta.9 → 0.10.0-rc.1

package/src/server/index.ts

@@ -1,103 +1,175 @@
 import cluster from 'cluster';
 import path from 'path';
+import sh from 'shelljs';
+import { getUserAgent } from 'package-manager-detector/detect';
+import { resolveCommand } from 'package-manager-detector/commands';
 import { readConfig, defaultBrowser } from './config.js';
-import { Options, Config, BrowserConfigObject, isWorkerMessage } from '../types.js';
+import {
+  Options,
+  Config,
+  BrowserConfigObject,
+  isWorkerMessage,
+  WorkerOptions,
+  OptionsSchema,
+  WorkerOptionsSchema,
+} from '../types.js';
 import { logger } from './logger.js';
+import { getStorybookUrl, checkIsStorybookConnected } from './connection.js';
 import { SeleniumWebdriver } from './selenium/webdriver.js';
 import { LOCALHOST_REGEXP } from './webdriver.js';
-import { isInsideDocker } from './utils.js';
-import { sendWorkerMessage } from './messages.js';
-import { playwrightDockerFile } from './playwright/docker-file.js';
+import { isInsideDocker, killTree, resolvePlaywrightBrowserType, shutdownWithError } from './utils.js';
+import { sendWorkerMessage, subscribeOn } from './messages.js';
 import { buildImage } from './docker.js';
-import { writeFile } from 'fs/promises';
+import { mkdir, writeFile } from 'fs/promises';
+import assert from 'assert';
+import * as v from 'valibot';
+import { PlaywrightWebdriver } from 'src/playwright.js';
+
+async function getPlaywrightVersion(): Promise<string> {
+  const {
+    default: { version },
+  } = await import('playwright-core/package.json', { with: { type: 'json' } });
+  return version;
+}
 
-async function startWebdriverServer(browser: string, config: Config, options: Options): Promise<string | undefined> {
-  if (config.webdriver === SeleniumWebdriver) {
-    if (cluster.isPrimary) {
-      const { startSelenoidContainer, startSelenoidStandalone } = await import('./selenium/selenoid.js');
-      const gridUrl = 'http://localhost:4444/wd/hub';
-      if (config.useDocker) {
-        const host = await startSelenoidContainer(config, options.debug);
-        return isInsideDocker ? gridUrl.replace(LOCALHOST_REGEXP, host) : gridUrl;
-      }
-      await startSelenoidStandalone(config, options.debug);
-      return gridUrl;
-    }
-    // TODO Worker might want to use docker image of browser or start standalone selenium
-  } else {
-    if (config.gridUrl) return undefined;
+async function startSelenoid(config: Config, debug = false): Promise<string | undefined> {
+  const { startSelenoidContainer, startSelenoidStandalone } = await import('./selenium/selenoid.js');
+  const gridUrl = 'http://localhost:4444/wd/hub';
+  if (config.useDocker) {
+    const host = await startSelenoidContainer(config, debug);
+    return isInsideDocker ? gridUrl.replace(LOCALHOST_REGEXP, host) : gridUrl;
+  }
+  await startSelenoidStandalone(config, debug);
+  return gridUrl;
+}
 
-    // TODO start standalone playwright server (useDocker == false)
-    const {
-      default: { version },
-    } = await import('playwright-core/package.json', { with: { type: 'json' } });
+async function startPlaywright(config: Config, browser: string, version: string, debug = false): Promise<string> {
+  // TODO Re-use dockerImage
+  const { startPlaywrightContainer } = await import('./playwright/docker.js');
+  const { browserName } = config.browsers[browser] as BrowserConfigObject;
 
-    if (cluster.isWorker) {
-      // TODO Re-use dockerImage
+  const imageName = `creevey/${browserName}:v${version}`;
+  const host = await startPlaywrightContainer(imageName, browser, config, debug);
 
-      // TODO Use https://hub.docker.com/r/playwright/chrome
-      // NOTE It will be possible to use `chrome` browserName
-      const { startPlaywrightContainer } = await import('./playwright/docker.js');
-      const { browserName } = config.browsers[browser] as BrowserConfigObject;
+  return host;
+}
 
+async function buildPlaywright(config: Config, version: string): Promise<void> {
+  const { playwrightDockerFile } = await import('./playwright/docker-file.js');
+  const {
+    default: { version: creeveyVersion },
+  } = await import('../../package.json', { with: { type: 'json' } });
+  const browsers = [...new Set(Object.values(config.browsers).map((c) => (c as BrowserConfigObject).browserName))];
+  await Promise.all(
+    browsers.map(async (browserName) => {
       const imageName = `creevey/${browserName}:v${version}`;
-      const host = await startPlaywrightContainer(imageName, options.debug);
+      const dockerfile = await playwrightDockerFile(browserName, version);
 
-      return host;
-    } else {
-      const browsers = [...new Set(Object.values(config.browsers).map((c) => (c as BrowserConfigObject).browserName))];
-      await Promise.all(
-        browsers.map(async (browserName) => {
-          const imageName = `creevey/${browserName}:v${version}`;
-          const dockerfile = playwrightDockerFile(browserName, version);
-
-          await buildImage(imageName, dockerfile);
-        }),
-      );
-
-      const { default: getPort } = await import('get-port');
-
-      cluster.on('message', (worker, message: unknown) => {
-        if (!isWorkerMessage(message)) return;
-
-        const workerMessage = message;
-        if (workerMessage.type != 'port') return;
-
-        void getPort().then((port) => {
-          sendWorkerMessage(worker, {
-            type: 'port',
-            payload: { port },
-          });
-        });
+      await buildImage(imageName, creeveyVersion, dockerfile);
+    }),
+  );
+
+  const { default: getPort } = await import('get-port');
+
+  cluster.on('message', (worker, message: unknown) => {
+    if (!isWorkerMessage(message)) return;
+
+    const workerMessage = message;
+    if (workerMessage.type != 'port') return;
+
+    void getPort().then((port) => {
+      sendWorkerMessage(worker, {
+        type: 'port',
+        payload: { port },
       });
+    });
+  });
+}
+
+async function startWebdriverServer(config: Config, options: Options): Promise<string | undefined> {
+  if (config.webdriver === SeleniumWebdriver) {
+    return startSelenoid(config, options.debug);
+    // TODO Worker might want to use docker image of browser or start standalone selenium
+  } else {
+    if (config.gridUrl) return undefined;
+
+    if (config.useDocker) {
+      const version = await getPlaywrightVersion();
+      await buildPlaywright(config, version);
     }
     // TODO Support gridUrl for playwright
     // NOTE: There is no grid for playwright right now
   }
 }
 
-export default async function (options: Options): Promise<void> {
+async function waitForStorybook(config: Config, options: Options): Promise<void> {
+  const [localUrl, remoteUrl] = getStorybookUrl(config, options);
+
+  if (options.storybookStart) {
+    const pm = getUserAgent();
+    assert(pm, new Error('Failed to detect current package manager'));
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const { command, args } = resolveCommand(pm, 'run', ['storybook', 'dev'])!;
+    const storybookPort = new URL(localUrl).port;
+    const storybookCommand = `${config.storybookAutorunCmd ?? [command, ...args, '--ci'].join(' ')} -p ${storybookPort}`;
+
+    logger().info(`Start Storybook via \`${storybookCommand}\`, it should be accessible at:`);
+    logger().info(`Local - ${localUrl}`);
+    if (remoteUrl && localUrl != remoteUrl) logger().info(`On your network - ${remoteUrl}`);
+    logger().info('Waiting Storybook...');
+
+    const storybook = sh.exec(storybookCommand, { async: true });
+    subscribeOn('shutdown', () => {
+      if (storybook.pid) void killTree(storybook.pid);
+    });
+  } else {
+    logger().info('Storybook should be started and be accessible at:');
+    logger().info(`Local - ${localUrl}`);
+    if (remoteUrl && localUrl != remoteUrl) logger().info(`On your network - ${remoteUrl}`);
+    logger().info(
+      'Tip: Creevey can start Storybook automatically by using `-s` option at the command line. (e.g., yarn/npm run creevey -s)',
+    );
+    logger().info('Waiting Storybook...');
+  }
+
+  if (options.storybookStart || process.env.CI !== 'true') {
+    const isConnected = await checkIsStorybookConnected(localUrl);
+    if (isConnected) {
+      logger().info('Storybook connected!\n');
+    } else {
+      logger().error('Storybook is not responding. Please start Storybook and restart Creevey');
+      shutdownWithError();
+    }
+  }
+}
+
+// TODO Why docker containers are not deleting after stop?
+export default async function (command: 'report' | 'test' | 'worker', options: Options | WorkerOptions): Promise<void> {
   const config = await readConfig(options);
-  const { browser = defaultBrowser, update, ui, port } = options;
-  let gridUrl = cluster.isPrimary ? config.gridUrl : options.gridUrl;
 
   // TODO Add package.json with `"type": "commonjs"` as workaround for esm packages to load `data.js`
+  await mkdir(config.reportDir, { recursive: true });
   await writeFile(path.join(config.reportDir, 'package.json'), '{"type": "commonjs"}');
 
-  // NOTE: We don't need docker nor selenoid for update option
-  if (
-    !(gridUrl || (Object.values(config.browsers) as BrowserConfigObject[]).every(({ gridUrl }) => gridUrl)) &&
-    !update
-  ) {
-    gridUrl = await startWebdriverServer(browser, config, options);
-  }
+  await import('./shutdown.js');
 
-  switch (true) {
-    case Boolean(update): {
-      (await import('./update.js')).update(config, typeof update == 'string' ? update : undefined);
+  if (v.is(OptionsSchema, options)) {
+    const { port, reportDir = config.reportDir } = options;
+
+    if (command == 'report') {
+      const { report } = await import('./report.js');
+      report(config, reportDir, port);
       return;
     }
-    case cluster.isPrimary: {
+
+    if (cluster.isPrimary) {
+      let gridUrl: string | undefined = config.gridUrl;
+
+      if (!(gridUrl || (Object.values(config.browsers) as BrowserConfigObject[]).every(({ gridUrl }) => gridUrl))) {
+        gridUrl = await startWebdriverServer(config, options);
+      }
+      await waitForStorybook(config, options);
+
       if (config.webdriver === SeleniumWebdriver) {
         try {
           await import('selenium-webdriver');
@@ -115,16 +187,30 @@ export default async function (options: Options): Promise<void> {
       }
       logger().info('Starting Master Process');
 
-      const resolveApi = (await import('./master/server.js')).start(config.reportDir, port, ui);
-
-      return (await import('./master/start.js')).start(gridUrl, config, options, resolveApi);
+      return (await import('./master/start.js')).start(gridUrl, config, options);
     }
-    default: {
-      logger().info(`Starting Worker for ${browser}`);
+  }
 
-      // NOTE: We assume that we pass `gridUrl` to worker CLI options
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      return (await import('./worker/start.js')).start(browser, gridUrl!, config, options);
+  if (v.is(WorkerOptionsSchema, options) && cluster.isWorker) {
+    let gridUrl = options.gridUrl;
+    const { browser = defaultBrowser, debug } = options;
+
+    if (!gridUrl) {
+      if (config.webdriver === PlaywrightWebdriver) {
+        if (config.useDocker) {
+          const version = await getPlaywrightVersion();
+          gridUrl = await startPlaywright(config, browser, version, debug);
+        } else {
+          const { browserName } = config.browsers[browser] as BrowserConfigObject;
+          gridUrl = `creevey://${resolvePlaywrightBrowserType(browserName)}`;
+        }
+      } else {
+        assert(gridUrl, 'Grid URL is required for Selenium');
+      }
     }
+
+    logger().info(`Starting Worker for ${browser}`);
+
+    return (await import('./worker/start.js')).start(browser, gridUrl, config, options);
   }
 }

package/src/server/master/api.ts

@@ -1,14 +1,10 @@
-import WebSocket from 'ws';
-import Runner from './runner.js';
-import { Request, Response, CreeveyUpdate } from '../../types.js';
+import { Data, WebSocket, WebSocketServer } from 'ws';
+import type { Request, Response, CreeveyUpdate } from '../../types.js';
+import type { TestsManager } from './testsManager.js';
+import type Runner from './runner.js';
 import { logger } from '../logger.js';
 
-export interface CreeveyApi {
-  subscribe: (wss: WebSocket.Server) => void;
-  handleMessage: (ws: WebSocket, message: WebSocket.Data) => void;
-}
-
-function broadcast(wss: WebSocket.Server, message: Response): void {
+function broadcast(wss: WebSocketServer, message: Response): void {
   wss.clients.forEach((ws) => {
     if (ws.readyState === WebSocket.OPEN) {
       ws.send(JSON.stringify(message));
@@ -16,44 +12,114 @@ function broadcast(wss: WebSocket.Server, message: Response): void {
   });
 }
 
-export default function creeveyApi(runner: Runner): CreeveyApi {
-  return {
-    subscribe(wss: WebSocket.Server) {
-      runner.on('update', (payload: CreeveyUpdate) => {
-        broadcast(wss, { type: 'update', payload });
+function send(ws: WebSocket, message: Response): void {
+  if (ws.readyState === WebSocket.OPEN) {
+    ws.send(JSON.stringify(message));
+  }
+}
+
+// The class-based implementation of CreeveyApi for native WebSockets
+export class CreeveyApi {
+  private runner: Runner | null = null;
+  private testsManager: TestsManager;
+  private wss: WebSocketServer | null = null;
+
+  constructor(testsManager: TestsManager, runner?: Runner) {
+    this.testsManager = testsManager;
+
+    // Use the provided runner in normal mode, or keep as null in update mode
+    if (runner) {
+      this.runner = runner;
+    }
+  }
+
+  subscribe(wss: WebSocketServer): void {
+    this.wss = wss;
+
+    // If we have a runner, subscribe to its updates
+    if (this.runner) {
+      this.runner.on('update', (payload: CreeveyUpdate) => {
+        this.broadcastUpdate(payload);
+      });
+    } else {
+      // Subscribe to TestsManager updates
+      this.testsManager.on('update', (update: CreeveyUpdate) => {
+        this.broadcastUpdate(update);
       });
-    },
+    }
+  }
 
-    handleMessage(ws: WebSocket, message: WebSocket.Data) {
-      if (typeof message != 'string') {
-        logger().info('unhandled message', message);
-        return;
-      }
+  handleMessage(ws: WebSocket, message: Data): void {
+    if (typeof message != 'string') {
+      logger().info('unhandled message', message);
+      return;
+    }
 
-      const command = JSON.parse(message) as Request;
+    const command = JSON.parse(message) as Request;
 
+    if (this.runner) {
+      // Normal mode handling with runner
       switch (command.type) {
         case 'status': {
-          ws.send(JSON.stringify({ type: 'status', payload: runner.status }));
+          const status = this.runner.status;
+          send(ws, { type: 'status', payload: status });
           return;
         }
         case 'start': {
-          runner.start(command.payload);
+          this.runner.start(command.payload);
           return;
         }
         case 'stop': {
-          runner.stop();
+          this.runner.stop();
+          return;
+        }
+        case 'approve': {
+          void this.runner.approve(command.payload);
+          return;
+        }
+        case 'approveAll': {
+          void this.runner.approveAll();
           return;
         }
+      }
+    } else {
+      // In update mode, only approve and approveAll commands are allowed
+      switch (command.type) {
         case 'approve': {
-          void runner.approve(command.payload);
+          void this.testsManager.approve(command.payload);
           return;
         }
         case 'approveAll': {
-          void runner.approveAll();
+          void this.testsManager.approveAll();
+          return;
+        }
+        case 'status': {
+          // In update mode, respond with static status including tests data
+          send(ws, {
+            type: 'status',
+            payload: {
+              isRunning: false,
+              tests: this.testsManager.getTestsData(),
+              browsers: [],
+              isUpdateMode: true,
+            },
+          });
+          return;
+        }
+        default: {
+          // Ignore other commands in update mode
+          logger().debug(`Command ${command.type} is not available in update mode`);
           return;
         }
       }
-    },
-  };
+    }
+  }
+
+  private broadcastUpdate(payload: CreeveyUpdate): void {
+    if (!this.wss) return;
+
+    const message: Response = { type: 'update', payload };
+
+    broadcast(this.wss, message);
+  }
 }

package/src/server/master/handlers/capture-handler.ts

@@ -0,0 +1,20 @@
+import cluster from 'cluster';
+import { subscribeOnWorker, sendStoriesMessage } from '../../messages.js';
+import { CaptureOptions, isDefined } from '../../../types.js';
+
+export function captureHandler({ workerId, options }: { workerId: number; options?: CaptureOptions }): void {
+  const worker = Object.values(cluster.workers ?? {})
+    .filter(isDefined)
+    .find((worker) => worker.process.pid == workerId);
+
+  // NOTE: Hypothetical case when someone send to us capture req and we don't have a worker with browser session for it
+  if (!worker) {
+    return;
+  }
+
+  const unsubscribe = subscribeOnWorker(worker, 'stories', (message) => {
+    if (message.type != 'capture') return;
+    unsubscribe();
+  });
+  sendStoriesMessage(worker, { type: 'capture', payload: options });
+}

package/src/server/master/handlers/index.ts

@@ -0,0 +1,4 @@
+export * from './ping-handler.js';
+export * from './stories-handler.js';
+export * from './capture-handler.js';
+export * from './static-handler.js';

package/src/server/master/handlers/ping-handler.ts

@@ -0,0 +1,6 @@
+import { IncomingMessage, ServerResponse } from 'http';
+
+export function pingHandler(_request: IncomingMessage, response: ServerResponse): void {
+  response.setHeader('Content-Type', 'text/plain');
+  response.end('pong');
+}

package/src/server/master/handlers/static-handler.ts

@@ -0,0 +1,16 @@
+import path from 'path';
+import fs from 'fs';
+
+export function staticHandler(baseDir: string, pathPrefix?: string) {
+  return (requestedPath: string): string | undefined => {
+    const relativePath = pathPrefix ? requestedPath.replace(pathPrefix, '') : requestedPath;
+    let filePath = path.join(baseDir, relativePath || 'index.html');
+
+    // If the path points to a directory, append index.html
+    if (fs.existsSync(filePath) && fs.statSync(filePath).isDirectory()) {
+      filePath = path.join(filePath, 'index.html');
+    }
+
+    return fs.existsSync(filePath) ? filePath : undefined;
+  };
+}

package/src/server/master/handlers/stories-handler.ts

@@ -0,0 +1,20 @@
+import cluster from 'cluster';
+import { emitStoriesMessage, sendStoriesMessage } from '../../messages.js';
+import { isDefined, StoryInput } from '../../../types.js';
+import { deserializeStory } from '../../../shared/index.js';
+
+export function storiesHandler({ stories }: { stories: [string, StoryInput[]][] }): void {
+  const deserializedStories = stories.map<[string, StoryInput[]]>(([file, stories]) => [
+    file,
+    stories.map(deserializeStory),
+  ]);
+
+  emitStoriesMessage({ type: 'update', payload: deserializedStories });
+
+  Object.values(cluster.workers ?? {})
+    .filter(isDefined)
+    .filter((worker) => worker.isConnected())
+    .forEach((worker) => {
+      sendStoriesMessage(worker, { type: 'update', payload: deserializedStories });
+    });
+}

package/src/server/master/master.ts

@@ -1,44 +1,27 @@
-import path from 'path';
-import { Config, TestData, isDefined, ServerTest } from '../../types.js';
-import { loadTestsFromStories, saveTestsJson } from '../stories.js';
+import { Config } from '../../types.js';
+import { loadTestsFromStories } from '../stories.js';
+import { TestsManager } from './testsManager.js';
 import Runner from './runner.js';
-import { tryToLoadTestsData } from '../utils.js';
-
-function mergeTests(
-  testsWithReports: Partial<Record<string, TestData>>,
-  testsFromStories: Partial<Record<string, ServerTest>>,
-): Partial<Record<string, ServerTest>> {
-  Object.values(testsFromStories)
-    .filter(isDefined)
-    .forEach((test) => {
-      const testWithReport = testsWithReports[test.id];
-      if (!testWithReport) return;
-      test.retries = testWithReport.retries;
-      if (testWithReport.status == 'success' || testWithReport.status == 'failed') test.status = testWithReport.status;
-      test.results = testWithReport.results;
-      test.approved = testWithReport.approved;
-    });
-  return testsFromStories;
-}
 
 export default async function master(config: Config, gridUrl?: string): Promise<Runner> {
-  const runner = new Runner(config, gridUrl);
-  const reportDataPath = path.join(config.reportDir, 'data.js');
-  const testsFromReport = tryToLoadTestsData(reportDataPath) ?? {};
+  // Create TestsManager instance
+  const testsManager = new TestsManager(config.screenDir, config.reportDir);
+
+  // Create Runner with TestsManager
+  const runner = new Runner(config, testsManager, gridUrl);
 
   await runner.init();
 
+  // Load tests from stories and update TestsManager
   const tests = await loadTestsFromStories(
     Object.keys(config.browsers),
     (listener) => config.storiesProvider(config, listener),
     (testsDiff) => {
       runner.updateTests(testsDiff);
-      saveTestsJson(runner.tests, config.reportDir);
     },
   );
 
-  runner.tests = mergeTests(testsFromReport, tests);
-  saveTestsJson(runner.tests, config.reportDir);
+  testsManager.updateTests(testsManager.loadAndMergeTests(tests));
 
   return runner;
 }

package/src/server/master/pool.ts

@@ -19,6 +19,7 @@ export default class Pool extends EventEmitter {
   private forcedStop = false;
   private failFast: boolean;
   private gridUrl?: string;
+  private storybookUrl: string;
   public get isRunning(): boolean {
     return this.workers.length !== this.freeWorkers.length;
   }
@@ -34,13 +35,16 @@ export default class Pool extends EventEmitter {
     this.maxRetries = config.maxRetries;
     this.config = config.browsers[browser] as BrowserConfigObject;
     this.gridUrl = this.config.gridUrl ?? gridUrl;
+    this.storybookUrl = this.config.storybookUrl ?? config.storybookUrl;
   }
 
   async init(): Promise<void> {
     const poolSize = Math.max(1, this.config.limit ?? 1);
     this.workers = (
       await Promise.all(
-        Array.from({ length: poolSize }).map(() => this.scheduler.forkWorker(this.browser, this.gridUrl)),
+        Array.from({ length: poolSize }).map(() =>
+          this.scheduler.forkWorker(this.browser, this.storybookUrl, this.gridUrl),
+        ),
       )
     ).filter((workerOrError): workerOrError is Worker => workerOrError instanceof ClusterWorker);
     if (this.workers.length != poolSize)
@@ -120,7 +124,7 @@ export default class Pool extends EventEmitter {
     worker.once('exit', async () => {
       if (isShuttingDown.current) return;
 
-      const workerOrError = await this.scheduler.forkWorker(this.browser, this.gridUrl);
+      const workerOrError = await this.scheduler.forkWorker(this.browser, this.storybookUrl, this.gridUrl);
 
       if (!(workerOrError instanceof ClusterWorker))
         throw new Error(`Can't instantiate worker for ${this.browser} due many errors`);
@@ -165,7 +169,7 @@ export default class Pool extends EventEmitter {
           gracefullyKill(worker);
         }
 
-        this.handleTestResult(worker, test, { status: 'failed', error: message.payload.error });
+        this.handleTestResult(worker, test, { status: 'failed', error: message.payload.error, retries: test.retries });
       }),
       subscribeOnWorker(worker, 'test', (message) => {
         if (message.type != 'end') return;