creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/src/creevey.ts

@@ -1,54 +1,108 @@
 import cluster from 'cluster';
-import minimist from 'minimist';
-import creevey from './server/index.js';
-import { Options } from './types.js';
-import { emitWorkerMessage } from './server/messages.js';
-import { isShuttingDown, shutdownWorkers } from './server/utils.js';
+import * as v from 'valibot';
 import Logger from 'loglevel';
+import { cac } from 'cac';
+import { Options, OptionsSchema, WorkerOptions, WorkerOptionsSchema } from './types.js';
+import { version } from '../package.json';
 import { logger, setRootName } from './server/logger.js';
+import creevey from './server/index.js';
+import './server/shutdown.js';
+
+const workerCli = cac('worker');
+workerCli
+  .command('worker', 'Start worker')
+  .option('--browser <browser>', 'Specify browser to run tests')
+  .option('--grid-url <url>', 'Selenium grid URL')
+  .option('-d, --debug', 'Enable debug mode')
+  .option('-c, --config <config>', 'Path to config file')
+  .option('-p, --port <port>', 'Port for UI server', { default: 3000 })
+  .option('--trace', 'Enable trace mode (more verbose than debug)')
+  .option('--report-dir <dir>', 'Directory for test reports')
+  .option('--screen-dir <dir>', 'Directory for reference images')
+  .option('--storybook-url <url>', 'Storybook server URL')
+  .option('--odiff', 'Use odiff for image comparison');
+workerCli.parse();
 
-function shutdownOnException(reason: unknown): void {
-  if (isShuttingDown.current) return;
+const cli = cac('creevey');
 
-  const error = reason instanceof Error ? (reason.stack ?? reason.message) : (reason as string);
+cli
+  .command('report [reportDir]', 'Launch web UI to review and approve test results')
+  .option('-c, --config <config>', 'Path to config file')
+  .option('-p, --port <port>', 'Port for UI server', { default: 3000 })
+  .option('--screen-dir <dir>', 'Directory for reference images');
 
-  logger().error(error);
+// TODO Add ability to start specific tests/files/stories
+cli
+  .command('test [options]', 'Run tests')
+  .option('--ui', 'Launch web UI for running tests and reviewing test results')
+  .option('-s, --storybook-start [cmd]', 'Start Storybook automatically')
+  .option('-c, --config <config>', 'Path to config file')
+  .option('-d, --debug', 'Enable debug mode')
+  .option('-p, --port <port>', 'Port for UI server', { default: 3000 })
+  .option('--fail-fast', 'Stop tests after first failure')
+  .option('--report-dir <dir>', 'Directory for test reports')
+  .option('--screen-dir <dir>', 'Directory for reference images')
+  .option('--storybook-url <url>', 'Storybook server URL')
+  .option('--storybook-port <port>', 'Storybook server port')
+  .option('--reporter <reporter>', '[DEPRECATED] Use config file instead')
+  .option('--odiff', 'Use odiff for image comparison')
+  .option('--trace', 'Enable trace mode (more verbose than debug)')
+  .option('--no-docker', 'Disable Docker usage');
+cli.version(process.env.npm_package_version ?? version);
+cli.help();
+cli.parse();
 
-  process.exitCode = -1;
-  if (cluster.isWorker) emitWorkerMessage({ type: 'error', payload: { subtype: 'unknown', error } });
-  if (cluster.isPrimary) void shutdownWorkers();
+if (
+  process.argv.includes('--help') ||
+  process.argv.includes('-h') ||
+  process.argv.includes('--version') ||
+  process.argv.includes('-v')
+) {
+  process.exit(0);
 }
 
-process.on('uncaughtException', shutdownOnException);
-process.on('unhandledRejection', shutdownOnException);
-// TODO SIGINT Stuck with selenium
-process.on('SIGINT', () => {
-  if (isShuttingDown.current) {
-    process.exit(-1);
+const command = cluster.isWorker ? workerCli.matchedCommandName : cli.matchedCommandName;
+const args = cluster.isWorker ? workerCli.args : cli.args;
+let options: Options | WorkerOptions;
+
+if (!command || (command !== 'report' && command !== 'test' && command !== 'worker')) {
+  console.error('Error: No known command specified\n');
+  cli.outputHelp();
+  process.exit(1);
+}
+
+try {
+  options = cluster.isWorker ? v.parse(WorkerOptionsSchema, workerCli.options) : v.parse(OptionsSchema, cli.options);
+} catch (error: unknown) {
+  if (v.isValiError(error)) {
+    console.error('Options validation failed:');
+    for (const issue of error.issues) {
+      const path = issue.path?.map((p) => p.key).join('.');
+      console.error(`  ${path ? `${path}: ` : ''}${issue.message}`);
+    }
+  } else {
+    console.error(error);
   }
-  isShuttingDown.current = true;
-});
+  console.log();
+  cli.matchedCommand?.outputHelp();
+
+  process.exit(1);
+}
 
-const argv = minimist<Options>(process.argv.slice(2), {
-  string: ['browser', 'config', 'reporter', 'reportDir', 'screenDir', 'gridUrl', 'storybookUrl'],
-  boolean: ['debug', 'trace', 'ui', 'odiff'],
-  default: { port: 3000 },
-  alias: { port: 'p', config: 'c', debug: 'd', update: 'u' },
-});
+if (v.is(OptionsSchema, options) && command == 'report' && args.length > 0) {
+  options.reportDir = args[0];
+  options.ui = true;
+}
 
-if ('port' in argv && !isNaN(argv.port)) argv.port = Number(argv.port);
-if ('browser' in argv && argv.browser) setRootName(argv.browser);
+// Handle browser name for logging
+if (v.is(WorkerOptionsSchema, options)) setRootName(options.browser);
 
-// eslint-disable-next-line @typescript-eslint/no-deprecated
-if (cluster.isPrimary && argv.reporter) {
+if (cluster.isPrimary && 'reporter' in options) {
   logger().warn(`--reporter option has been removed please describe reporter in config file:
     import { reporters } from 'mocha';
 
     const config = {
-      reporter: reporters.${
-        // eslint-disable-next-line @typescript-eslint/no-deprecated
-        argv.reporter
-      },
+      reporter: reporters.${options.reporter},
     };
 
     export default config;
@@ -56,11 +110,11 @@ if (cluster.isPrimary && argv.reporter) {
 }
 
 // @ts-expect-error: define log level for storybook
-global.LOGLEVEL = argv.trace ? 'trace' : argv.debug ? 'debug' : 'warn';
-if (argv.trace) {
+global.LOGLEVEL = options.trace ? 'trace' : options.debug ? 'debug' : 'warn';
+if (options.trace) {
   logger().setDefaultLevel(Logger.levels.TRACE);
   Logger.setDefaultLevel(Logger.levels.TRACE);
-} else if (argv.debug) {
+} else if (options.debug) {
   logger().setDefaultLevel(Logger.levels.DEBUG);
   Logger.setDefaultLevel(Logger.levels.DEBUG);
 } else {
@@ -68,4 +122,4 @@ if (argv.trace) {
   Logger.setDefaultLevel(Logger.levels.INFO);
 }
 
-void creevey(argv);
+void creevey(command, options);

package/src/playwright/generator.ts

@@ -0,0 +1,322 @@
+import path from 'path';
+import assert from 'assert';
+import { readFileSync } from 'fs';
+import type { PixelmatchOptions } from 'pixelmatch';
+import type { ODiffOptions } from 'odiff-bin';
+import { test, Page } from '@playwright/test';
+import isEqual from 'lodash/isEqual.js';
+import { CreeveyStoryParams, isObject, StoriesRaw, StorybookEvents, StorybookGlobals } from '../types';
+import { getOdiffAssert, getPixelmatchAssert, ImageContext } from '../server/compare';
+import { appendIframePath } from '../server/webdriver';
+import { waitForStorybookReady } from './helpers';
+
+export interface TestsConfig {
+  /**
+   * Define pixelmatch diff options
+   * @default { threshold: 0.1, includeAA: false }
+   */
+  diffOptions: PixelmatchOptions;
+  /**
+   * Define odiff diff options
+   * @default { threshold: 0.1, antialiasing: true }
+   */
+  odiffOptions: ODiffOptions;
+  /**
+   * Define matcher for visual regression assertion
+   * @default 'pixelmatch'
+   */
+  comparisonLibrary: 'pixelmatch' | 'odiff';
+  /**
+   * Enables page context reuse across tests for faster execution, though this breaks test isolation.
+   * @default true
+   */
+  reusePageContext: boolean;
+}
+
+const cacheDir = process.env.CREEVEY_CACHE_DIR;
+const defaultConfig: TestsConfig = {
+  diffOptions: { threshold: 0.1, includeAA: false },
+  odiffOptions: { threshold: 0.1, antialiasing: true },
+  comparisonLibrary: 'pixelmatch',
+  reusePageContext: true,
+};
+
+// TODO: Use this Storybook function for building args for query params
+// export const buildArgsParam = (initialArgs: Args | undefined, args: Args): string => {
+
+// TODO: Pass globals to story
+function appendStoryQueryParams(url: string, storyId: string): string {
+  return `${url}?args=&globals=&id=${storyId}`;
+}
+
+function assertWrapper(
+  assert: (actual: Buffer, imageName?: string) => Promise<string | undefined>,
+): (actual: Buffer, imageName?: string) => Promise<void> {
+  return async function assertImage(actual, imageName) {
+    try {
+      const errorMessage = await assert(actual, imageName);
+      if (errorMessage) {
+        throw new Error(errorMessage);
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        error.stack = error.stack
+          ?.split('\n')
+          .filter((line) => !line.includes('at assertImage'))
+          .join('\n');
+      }
+      throw error;
+    }
+  };
+}
+
+async function takeScreenshot(
+  page: Page,
+  storyId: string,
+  captureElement?: string | null,
+  ignoreElements?: string | string[] | null,
+): Promise<Buffer> {
+  const ignore = ignoreElements ? (Array.isArray(ignoreElements) ? ignoreElements : [ignoreElements]) : [];
+  const mask = ignore.map((selector) => page.locator(selector));
+
+  if (captureElement) {
+    const element = await page.$(captureElement);
+    if (!element) throw new Error(`Capture element '${captureElement}' not found for story '${storyId}'`);
+    return element.screenshot({
+      style: ':root { overflow: hidden !important; }',
+      animations: 'disabled',
+      mask,
+    });
+  } else {
+    return page.screenshot({
+      animations: 'disabled',
+      mask,
+    });
+  }
+}
+
+// TODO: To support parallel tests, we need to define each test suite in separate file
+// TODO: How to support custom interactions for different tests
+// Main function to define tests using Playwright's API
+export function definePlaywrightTests(config?: Partial<TestsConfig>): void {
+  assert(cacheDir, 'Cache directory not found');
+
+  const stories = JSON.parse(readFileSync(path.join(cacheDir, 'stories.json'), 'utf-8')) as StoriesRaw;
+  let globals: StorybookGlobals = {};
+  let reusedPage: Page;
+
+  const { diffOptions, odiffOptions, comparisonLibrary, reusePageContext } = {
+    ...defaultConfig,
+    ...config,
+  };
+
+  async function updateGlobals(page: Page, storybookGlobals: unknown): Promise<void> {
+    if (storybookGlobals && typeof storybookGlobals === 'object' && !isEqual(globals, storybookGlobals)) {
+      globals = storybookGlobals as StorybookGlobals;
+      await page.evaluate((globals) => {
+        window.__STORYBOOK_ADDONS_CHANNEL__.emit(StorybookEvents.UPDATE_GLOBALS, { globals });
+      }, globals);
+    }
+  }
+
+  test.describe('Creevey Tests', () => {
+    const imagesContext: ImageContext = {
+      attachments: [],
+      testFullPath: [],
+      images: {},
+    };
+    let assertImage: (actual: Buffer, imageName?: string) => Promise<void>;
+
+    test.beforeAll('Setup images context', async ({ browser }, { project }) => {
+      const { snapshotDir, outputDir } = project;
+      if (reusePageContext) {
+        const storybookUrl = project.use.baseURL;
+
+        assert(storybookUrl, 'Storybook URL not found');
+
+        reusedPage = await browser.newPage();
+        await reusedPage.goto(appendIframePath(storybookUrl), { waitUntil: 'networkidle', timeout: 60000 });
+        await waitForStorybookReady(reusedPage);
+      }
+      if (comparisonLibrary === 'pixelmatch') {
+        const { default: pixelmatch } = await import('pixelmatch');
+        assertImage = assertWrapper(
+          getPixelmatchAssert(pixelmatch, imagesContext, { screenDir: snapshotDir, reportDir: outputDir, diffOptions }),
+        );
+      } else {
+        const { compare } = await import('odiff-bin');
+        assertImage = assertWrapper(
+          getOdiffAssert(compare, imagesContext, { screenDir: snapshotDir, reportDir: outputDir, odiffOptions }),
+        );
+      }
+    });
+
+    test.beforeEach('Switch story', async ({ page }, { annotations, project }) => {
+      const { description: storyId } = annotations.find((annotation) => annotation.type === 'storyId') ?? {};
+
+      assert(storyId, 'Cannot get storyId. It seems like inner test annotation is missing');
+
+      const story = stories[storyId];
+
+      assert(story, `Story '${storyId}' not found in stories cache`);
+
+      const { title, name, parameters } = story;
+      const { waitForReady: shouldWaitForReady } = (parameters.creevey ?? {}) as CreeveyStoryParams;
+
+      const storybookGlobals: unknown = project.metadata.storybookGlobals;
+
+      imagesContext.attachments = [];
+      imagesContext.testFullPath = [...title.split('/').map((x) => x.trim()), name, project.name];
+      imagesContext.images = {};
+
+      if (!reusePageContext) {
+        const storybookUrl = project.use.baseURL;
+
+        assert(storybookUrl, 'Storybook URL not found');
+
+        await page.goto(appendStoryQueryParams(appendIframePath(storybookUrl), storyId), {
+          waitUntil: 'networkidle',
+          timeout: 60000,
+        });
+        await waitForStorybookReady(page);
+        // TODO: Pass globals to story
+        await updateGlobals(page, storybookGlobals);
+
+        return;
+      }
+
+      // 1. Update Storybook Globals
+      await updateGlobals(reusedPage, storybookGlobals);
+
+      // 2. Reset Mouse Position
+      await reusedPage.mouse.move(0, 0);
+
+      // 3. Select Story
+      const errorMessage = await reusedPage.evaluate<
+        string | null,
+        { storyId: string; StorybookEvents: typeof StorybookEvents; shouldWaitForReady?: boolean }
+      >(
+        async ({ storyId, StorybookEvents, shouldWaitForReady }) => {
+          // TODO: DRY with withCreevey.ts
+          // NOTE: Copy-pasted from withCreevey.ts
+          const channel = window.__STORYBOOK_ADDONS_CHANNEL__;
+
+          const waitForReady = shouldWaitForReady
+            ? new Promise<void>((resolve) => (window.__CREEVEY_SET_READY_FOR_CAPTURE__ = resolve))
+            : Promise.resolve();
+
+          let rejectCallback: (reason?: unknown) => void;
+          const renderErrorPromise = new Promise<void>((_resolve, reject) => (rejectCallback = reject));
+
+          function errorHandler({ title, description }: { title: string; description: string }): void {
+            rejectCallback({
+              message: title,
+              stack: description,
+            });
+          }
+          function exceptionHandler(exception: Error): void {
+            rejectCallback(exception);
+          }
+          function removeErrorHandlers(): void {
+            channel.off(StorybookEvents.STORY_ERRORED, errorHandler);
+            channel.off(StorybookEvents.STORY_THREW_EXCEPTION, errorHandler);
+          }
+
+          channel.once(StorybookEvents.STORY_ERRORED, errorHandler);
+          channel.once(StorybookEvents.STORY_THREW_EXCEPTION, exceptionHandler);
+
+          let resolveCallback: () => void;
+          const storyRenderedPromise = new Promise<void>((resolve) => (resolveCallback = resolve));
+          function renderHandler(): void {
+            resolveCallback();
+          }
+          function removeRenderHandlers(): void {
+            channel.off(StorybookEvents.STORY_RENDERED, renderHandler);
+          }
+
+          channel.once(StorybookEvents.STORY_RENDERED, renderHandler);
+
+          setTimeout(() => {
+            channel.emit(StorybookEvents.SET_CURRENT_STORY, { storyId });
+          }, 0);
+
+          try {
+            await Promise.race([renderErrorPromise, Promise.all([storyRenderedPromise, waitForReady])]);
+          } catch (reason) {
+            // NOTE Event `STORY_THREW_EXCEPTION` triggered only in react and vue frameworks and return Error instance
+            // NOTE Event `STORY_ERRORED` return error-like object without `name` field
+            const errorMessage =
+              reason instanceof Error
+                ? (reason.stack ?? reason.message)
+                : isObject(reason)
+                  ? `${reason.message as string}\n    ${reason.stack as string}`
+                  : (reason as string);
+            return errorMessage;
+          } finally {
+            removeErrorHandlers();
+            removeRenderHandlers();
+          }
+
+          return null;
+        },
+        { storyId: story.id, StorybookEvents, shouldWaitForReady },
+      );
+
+      if (errorMessage) {
+        throw new Error(`Failed to select story '${story.id}': ${errorMessage}`);
+      }
+    });
+
+    test.afterEach('Save screenshot', () => {
+      const { name: projectName } = test.info().project;
+
+      // TODO: Use another way to handle attachments
+
+      // NOTE: Don't need to copy files for assertImage, because it's done internally
+      const { actual, diff, expect } = imagesContext.images[projectName] ?? {};
+      for (const image of imagesContext.attachments) {
+        switch (true) {
+          case image.includes('actual') && !!actual: {
+            test.info().attachments.push({ name: actual, path: image, contentType: 'image/png' });
+            // await test.info().attach(actual, { path: image });
+            break;
+          }
+          case image.includes('expect') && !!expect: {
+            test.info().attachments.push({ name: expect, path: image, contentType: 'image/png' });
+            // await test.info().attach(expect, { path: image });
+            break;
+          }
+          case image.includes('diff') && !!diff: {
+            test.info().attachments.push({ name: diff, path: image, contentType: 'image/png' });
+            // await test.info().attach(diff, { path: image });
+            break;
+          }
+        }
+      }
+    });
+
+    for (const story of Object.values(stories)) {
+      const { name, title, parameters } = story;
+      const { captureElement, ignoreElements } = (parameters.creevey ?? {}) as CreeveyStoryParams;
+
+      test.describe(title, () => {
+        // TODO: Support creevey.skip
+        test(name, { annotation: [{ type: 'storyId', description: story.id }] }, async ({ page }) => {
+          // 4. Take Screenshot
+          const screenshot = await takeScreenshot(
+            reusePageContext ? reusedPage : page,
+            story.id,
+            captureElement,
+            ignoreElements,
+          );
+          // TODO: Support this
+          // NOTE: Bear in mind that page.locator('#root > *') is not working
+          // await expect(page.locator(captureElement)).toHaveScreenshot(name);
+
+          // 5. Assert Image
+          await assertImage(screenshot);
+        });
+      });
+    }
+  });
+}

package/src/playwright/helpers.ts

@@ -0,0 +1,31 @@
+import { StorybookEvents } from '../types';
+import { Page } from '@playwright/test';
+
+export async function waitForStorybookReady(page: Page, timeout = 60000): Promise<void> {
+  const isStorybookInitialized = await page.evaluate(
+    ({ timeout, event }: { timeout: number; event: string }) => {
+      return new Promise<boolean>((resolve, reject) => {
+        let attempts = 0;
+        const maxAttempts = timeout / 100;
+        function check() {
+          if (
+            typeof window.__STORYBOOK_ADDONS_CHANNEL__ !== 'undefined' &&
+            window.__STORYBOOK_ADDONS_CHANNEL__.last(event) !== undefined
+          ) {
+            resolve(true);
+          } else if (attempts++ < maxAttempts) {
+            setTimeout(check, 100);
+          } else {
+            reject(new Error('Storybook initialization timed out. Required Storybook functions not found on window.'));
+          }
+        }
+        check();
+      });
+    },
+    { timeout, event: StorybookEvents.SET_GLOBALS },
+  );
+
+  if (!isStorybookInitialized) {
+    throw new Error('Failed to confirm Storybook API is ready after extended wait.');
+  }
+}