creevey 0.10.0-beta.4 → 0.10.0-beta.40

package/src/server/worker/start.ts

@@ -1,18 +1,12 @@
 import chai from 'chai';
-import chalk from 'chalk';
-import Logger from 'loglevel';
-import EventEmitter from 'events';
 import {
   BaseCreeveyTestContext,
   Config,
   CreeveyWebdriver,
-  FakeSuite,
-  FakeTest,
-  Images,
   Options,
   ServerTest,
-  TEST_EVENTS,
   TestMessage,
+  TestResult,
   isDefined,
   isImageError,
 } from '../../types.js';
@@ -47,9 +41,10 @@ async function getTestsFromStories(
   return testsById;
 }
 
-function runHandler(browserName: string, images: Partial<Record<string, Images>>, error?: unknown): void {
+function runHandler(browserName: string, result: Omit<TestResult, 'status'>, error?: unknown): void {
   // TODO How handle browser corruption?
-  if (isImageError(error)) {
+  const { images } = result;
+  if (images != null && isImageError(error)) {
     if (typeof error.images == 'string') {
       const image = images[browserName];
       if (image) image.error = error.images;
@@ -62,31 +57,37 @@ function runHandler(browserName: string, images: Partial<Record<string, Images>>
     }
   }
 
-  if (error || Object.values(images).some((image) => image?.error != null)) {
-    const errorMessage = serializeError(error);
+  if (error || (images != null && Object.values(images).some((image) => image?.error != null))) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const errorMessage = result.error!;
 
     const isUnexpectedError =
       hasTimeout(errorMessage) ||
       hasDisconnected(errorMessage) ||
-      Object.values(images).some((image) => hasTimeout(image?.error));
+      (images != null && Object.values(images).some((image) => hasTimeout(image?.error)));
     if (isUnexpectedError) emitWorkerMessage({ type: 'error', payload: { subtype: 'unknown', error: errorMessage } });
     else
       emitTestMessage({
         type: 'end',
         payload: {
           status: 'failed',
-          images,
-          error: errorMessage,
+          ...result,
         },
       });
   } else {
-    emitTestMessage({ type: 'end', payload: { status: 'success', images } });
+    emitTestMessage({
+      type: 'end',
+      payload: {
+        status: 'success',
+        ...result,
+      },
+    });
   }
 }
 
 async function setupWebdriver(webdriver: CreeveyWebdriver): Promise<[string, CreeveyWebdriver] | undefined> {
   if ((await webdriver.openBrowser(true)) == null) {
-    logger.error('Failed to start browser');
+    logger().error('Failed to start browser');
     emitWorkerMessage({
       type: 'error',
       payload: { subtype: 'browser', error: 'Failed to start browser' },
@@ -114,7 +115,6 @@ function hasTimeout(str: string | null | undefined): boolean {
 }
 
 export async function start(browser: string, gridUrl: string, config: Config, options: Options): Promise<void> {
-  let retries = 0;
   const imagesContext: ImageContext = {
     attachments: [],
     testFullPath: [],
@@ -125,38 +125,16 @@ export async function start(browser: string, gridUrl: string, config: Config, op
 
   if (!webdriver || !sessionId) return;
 
-  const workerLogger = Logger.getLogger(`${browser}:${chalk.gray(sessionId)}`);
-
-  const reporterOptions = {
-    ...config.reporterOptions,
-    creevey: {
-      sessionId,
-      reportDir: config.reportDir,
-      browserName: browser,
-      get willRetry() {
-        return retries < config.maxRetries;
-      },
-      get images() {
-        return imagesContext.images;
-      },
-    },
-  };
-
-  class FakeRunner extends EventEmitter {}
-  const runner = new FakeRunner();
-  const Reporter = config.reporter;
-  new Reporter(runner, { reporterOptions });
-
   const { matchImage, matchImages } = options.odiff
     ? getOdiffMatchers(imagesContext, config)
     : await getMatchers(imagesContext, config);
-  chai.use(chaiImage(matchImage, matchImages, workerLogger));
+  chai.use(chaiImage(matchImage, matchImages));
 
   const tests = await (async () => {
     try {
       return await getTestsFromStories(config, browser, webdriver);
     } catch (error) {
-      workerLogger.error('Failed to get tests from stories:', error);
+      logger().error('Failed to get tests from stories:', error);
       emitWorkerMessage({
         type: 'error',
         payload: { subtype: 'browser', error: serializeError(error) },
@@ -174,7 +152,7 @@ export async function start(browser: string, gridUrl: string, config: Config, op
 
     if (!test) {
       const error = `Test with id ${message.payload.id} not found`;
-      workerLogger.error(error);
+      logger().error(error);
       emitWorkerMessage({
         type: 'error',
         payload: { subtype: 'test', error },
@@ -200,68 +178,53 @@ export async function start(browser: string, gridUrl: string, config: Config, op
     imagesContext.testFullPath = getTestPath(test);
     imagesContext.images = {};
 
-    retries = message.payload.retries;
     let error = undefined;
 
-    const fakeSuite: FakeSuite = {
-      title: test.storyPath.slice(0, -1).join('/'),
-      fullTitle: () => fakeSuite.title,
-      titlePath: () => [fakeSuite.title],
-      tests: [],
-    };
-
-    const fakeTest: FakeTest = {
-      parent: fakeSuite,
-      title: [test.story.name, test.testName, test.browser].filter(isDefined).join('/'),
-      fullTitle: () => getTestPath(test).join('/'),
-      titlePath: () => getTestPath(test),
-      currentRetry: () => retries,
-      retires: () => config.maxRetries,
-      slow: () => 1000,
-    };
-
-    fakeSuite.tests.push(fakeTest);
-
     void (async () => {
-      runner.emit(TEST_EVENTS.RUN_BEGIN);
-      runner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
-
+      let timeout;
+      let isRejected = false;
       const start = Date.now();
       try {
         await Promise.race([
-          new Promise((reject) =>
-            setTimeout(() => {
-              reject(`Timeout of ${config.testTimeout}ms exceeded`);
-            }, config.testTimeout),
+          new Promise(
+            (_, reject) =>
+              (timeout = setTimeout(() => {
+                isRejected = true;
+                reject(new Error(`Timeout of ${config.testTimeout}ms exceeded`));
+              }, config.testTimeout)),
           ),
           (async () => {
-            const context = await webdriver.switchStory(test.story, baseContext, workerLogger);
+            const context = await webdriver.switchStory(test.story, baseContext);
             await test.fn(context);
           })(),
         ]);
       } catch (testError) {
         error = testError;
-        fakeTest.err = error;
       }
       const duration = Date.now() - start;
-      fakeTest.attachments = imagesContext.attachments;
-      fakeTest.state = error ? 'failed' : 'passed';
-      fakeTest.duration = duration;
-      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
-
-      if (error) {
-        runner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, error);
-      } else {
-        runner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
-      }
-      runner.emit(TEST_EVENTS.TEST_END, fakeTest);
-      runner.emit(TEST_EVENTS.RUN_END);
+      clearTimeout(timeout);
 
       await webdriver.afterTest(test);
 
-      runHandler(baseContext.browserName, imagesContext.images, error);
+      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+      if (isRejected) {
+        emitWorkerMessage({
+          type: 'error',
+          payload: { subtype: 'unknown', error: serializeError(error) },
+        });
+      } else {
+        const result = {
+          sessionId,
+          images: imagesContext.images,
+          error: serializeError(error),
+          duration,
+          attachments: imagesContext.attachments,
+          retries: message.payload.retries,
+        };
+        runHandler(baseContext.browserName, result, error);
+      }
     })().catch((error: unknown) => {
-      workerLogger.error('Unexpected error:', error);
+      logger().error('Unexpected error:', error);
       emitWorkerMessage({
         type: 'error',
         payload: { subtype: 'test', error: serializeError(error) },
@@ -269,7 +232,7 @@ export async function start(browser: string, gridUrl: string, config: Config, op
     });
   });
 
-  workerLogger.info('Browser is ready');
+  logger().info('Browser is ready');
 
   emitWorkerMessage({ type: 'ready' });
 }

package/src/shared/index.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { Parameters } from '@storybook/csf';
+import type { Parameters } from '@storybook/types';
 import { SetStoriesData, StoriesRaw, CreeveyStoryParams, StoryInput } from '../types.js';
 import { deserializeRegExp, isSerializedRegExp, isRegExp, serializeRegExp } from './serializeRegExp.js';
 

package/src/types.ts

@@ -1,11 +1,10 @@
-import type { StoryContextForEnhancers, DecoratorFunction } from '@storybook/csf';
+import type { StoryContextForEnhancers, DecoratorFunction } from '@storybook/types';
 import type { Worker as ClusterWorker } from 'cluster';
 import type Pixelmatch from 'pixelmatch';
 import type { ODiffOptions } from 'odiff-bin';
 import type { expect } from 'chai';
 import type EventEmitter from 'events';
-import type Logger from 'loglevel';
-import { LaunchOptions } from 'playwright-core';
+import type { LaunchOptions } from 'playwright-core';
 // import type { Browser } from 'playwright-core';
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
@@ -103,6 +102,7 @@ export class ChromeConfig {
 }
 */
 export interface BrowserConfigObject {
+  // TODO Restrict browser names for playwright images
   browserName: string;
   // customizeBuilder?: (builder: Builder) => Builder;
   limit?: number;
@@ -145,7 +145,13 @@ export interface BrowserConfigObject {
     [name: string]: unknown;
   };
 
-  playwrightOptions?: Omit<LaunchOptions, 'logger'>;
+  playwrightOptions?: Omit<LaunchOptions, 'logger'> & {
+    trace?: {
+      screenshots?: boolean;
+      snapshots?: boolean;
+      sources?: boolean;
+    };
+  };
 }
 
 export type StorybookGlobals = Record<string, unknown>;
@@ -164,7 +170,7 @@ export interface CreeveyWebdriver {
   openBrowser(fresh?: boolean): Promise<CreeveyWebdriver | null>;
   closeBrowser(): Promise<void>;
   loadStoriesFromBrowser(): Promise<StoriesRaw>;
-  switchStory(story: StoryInput, context: BaseCreeveyTestContext, logger: Logger.Logger): Promise<CreeveyTestContext>;
+  switchStory(story: StoryInput, context: BaseCreeveyTestContext): Promise<CreeveyTestContext>;
   afterTest(test: ServerTest): Promise<void>;
 }
 
@@ -200,6 +206,11 @@ export interface Config {
    * Url where storybook hosted on
    */
   resolveStorybookUrl?: () => Promise<string>;
+  /**
+   * Command to automatically start Storybook if it is not running.
+   * For example, `npm run storybook`, `yarn run storybook` etc.
+   */
+  storybookAutorunCmd?: string;
   /**
    * Absolute path to directory with reference images
    * @default path.join(process.cwd(), './images')
@@ -307,6 +318,11 @@ export interface Config {
    * The `--ui` CLI option ignores this option
    */
   failFast: boolean;
+  /**
+   * Start workers in sequential queue
+   * @default false
+   */
+  useWorkerQueue: boolean;
   /**
    * Specify platform for docker images
    */
@@ -348,9 +364,13 @@ export interface Options {
   screenDir?: string;
   reportDir?: string;
   gridUrl?: string;
+  storybookStart?: boolean | string;
   storybookUrl?: string;
+  storybookPort?: string;
+  storybookAutorunCmd?: string;
   failFast?: boolean;
   odiff?: boolean;
+  noDocker?: boolean;
 }
 
 export type WorkerError = 'browser' | 'test' | 'unknown';
@@ -399,10 +419,15 @@ export type TestStatus = 'unknown' | 'pending' | 'running' | 'failed' | 'approve
 
 export interface TestResult {
   status: 'failed' | 'success';
+  retries: number;
   // TODO Remove checks `name == browser` in TestResultsView
   // images?: Partial<{ [name: string]: Images }> | Images;
   images?: Partial<Record<string, Images>>;
   error?: string;
+  // Test metadata for reporting
+  duration?: number;
+  attachments?: string[];
+  sessionId?: string;
 }
 
 export class ImagesError extends Error {
@@ -437,8 +462,8 @@ export interface BaseCreeveyTestContext {
    * @internal
    */
   screenshots: { imageName?: string; screenshot: Buffer }[];
-  matchImage: (image: Buffer, imageName?: string) => Promise<void>;
-  matchImages: (images: Record<string, Buffer>) => Promise<void>;
+  matchImage: (image: Buffer | string, imageName?: string) => Promise<void>;
+  matchImages: (images: Record<string, Buffer | string>) => Promise<void>;
 }
 
 export interface CreeveyTestContext extends BaseCreeveyTestContext {
@@ -450,6 +475,8 @@ export interface CreeveyTestContext extends BaseCreeveyTestContext {
 export enum TEST_EVENTS {
   RUN_BEGIN = 'start',
   RUN_END = 'end',
+  SUITE_BEGIN = 'suite',
+  SUITE_END = 'suite end',
   TEST_BEGIN = 'test',
   TEST_END = 'test end',
   TEST_FAIL = 'fail',
@@ -474,7 +501,7 @@ export interface FakeTest {
   title: string;
   fullTitle: () => string;
   titlePath: () => string[];
-  currentRetry: () => number;
+  currentRetry: () => number | undefined;
   retires: () => number;
   slow: () => number;
   duration?: number;
@@ -484,6 +511,15 @@ export interface FakeTest {
   err?: unknown;
   // NOTE: image files
   attachments?: string[];
+
+  // NOTE: Creevey specific fields
+  creevey: {
+    reportDir: string;
+    sessionId: string;
+    browserName: string;
+    willRetry: boolean;
+    images: Partial<Record<string, Partial<Images>>>;
+  };
 }
 
 export interface CreeveyStatus {

package/types/global.d.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-var */
 
+declare var __CREEVEY_ENV__: boolean;
 declare var __CREEVEY_SERVER_HOST__: string;
 declare var __CREEVEY_SERVER_PORT__: number | null;
 declare var __CREEVEY_CLIENT_PORT__: number | null;

package/.yarnrc.yml

@@ -1 +0,0 @@
-nodeLinker: node-modules

package/chromatic.config.json

@@ -1,5 +0,0 @@
-{
-  "onlyChanged": true,
-  "projectId": "Project:5fd73972015e660021b97564",
-  "zip": true
-}