creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/src/types.ts

@@ -1,24 +1,11 @@
-import type { StoryContextForEnhancers, DecoratorFunction } from '@storybook/csf';
+import * as v from 'valibot';
 import type { Worker as ClusterWorker } from 'cluster';
-import type Pixelmatch from 'pixelmatch';
-import type { ODiffOptions } from 'odiff-bin';
 import type { expect } from 'chai';
 import type EventEmitter from 'events';
-import { LaunchOptions } from 'playwright-core';
-// import type { Browser } from 'playwright-core';
-
-/* eslint-disable @typescript-eslint/no-explicit-any */
-export type DiffOptions = typeof Pixelmatch extends (
-  x1: any,
-  x2: any,
-  x3: any,
-  x4: any,
-  x5: any,
-  options?: infer T,
-) => void
-  ? T
-  : never;
-/* eslint-enable @typescript-eslint/no-explicit-any */
+import type { ODiffOptions } from 'odiff-bin';
+import type { LaunchOptions } from 'playwright-core';
+import type { PixelmatchOptions } from 'pixelmatch';
+import type { StoryContextForEnhancers, DecoratorFunction } from 'storybook/internal/types';
 
 export interface SetStoriesData {
   v?: number;
@@ -102,6 +89,7 @@ export class ChromeConfig {
 }
 */
 export interface BrowserConfigObject {
+  // TODO Restrict browser names for playwright images
   browserName: string;
   // customizeBuilder?: (builder: Builder) => Builder;
   limit?: number;
@@ -115,6 +103,10 @@ export interface BrowserConfigObject {
    * Storybook's globals to set in a specific browser
    * @see https://github.com/storybookjs/storybook/blob/v6.0.0/docs/essentials/toolbars-and-globals.md
    */
+  storybookGlobals?: StorybookGlobals;
+  /**
+   * @deprecated Use `storybookGlobals` instead
+   */
   _storybookGlobals?: StorybookGlobals;
   /**
    * Specify custom docker image. Used only with `useDocker == true`
@@ -144,7 +136,13 @@ export interface BrowserConfigObject {
     [name: string]: unknown;
   };
 
-  playwrightOptions?: Omit<LaunchOptions, 'logger'>;
+  playwrightOptions?: Omit<LaunchOptions, 'logger'> & {
+    trace?: {
+      screenshots?: boolean;
+      snapshots?: boolean;
+      sources?: boolean;
+    };
+  };
 }
 
 export type StorybookGlobals = Record<string, unknown>;
@@ -155,7 +153,7 @@ export type CreeveyWebdriverConstructor = new (
   browser: string,
   gridUrl: string,
   config: Config,
-  options: Options,
+  options: WorkerOptions,
 ) => CreeveyWebdriver;
 
 export interface CreeveyWebdriver {
@@ -182,7 +180,7 @@ export interface DockerAuth {
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type BaseReporter = new (runner: EventEmitter, options: { reporterOptions: any }) => void;
+export type BaseReporter = new (runner: EventEmitter, options: { reportDir: string; reporterOptions: any }) => void;
 
 export interface Config {
   /**
@@ -199,6 +197,11 @@ export interface Config {
    * Url where storybook hosted on
    */
   resolveStorybookUrl?: () => Promise<string>;
+  /**
+   * Command to automatically start Storybook if it is not running.
+   * For example, `npm run storybook` or `yarn storybook` etc.
+   */
+  storybookAutorunCmd?: string;
   /**
    * Absolute path to directory with reference images
    * @default path.join(process.cwd(), './images')
@@ -212,8 +215,9 @@ export interface Config {
   /**
    * Specify a custom reporter for test results. Creevey accepts only mocha-like reporters
    * @optional
+   * @default 'creevey'
    */
-  reporter: BaseReporter;
+  reporter: BaseReporter | 'creevey' | 'teamcity' | 'junit';
   /**
    * Options which are used by reporter
    */
@@ -231,13 +235,14 @@ export interface Config {
   testTimeout: number;
   /**
    * Define pixelmatch diff options
-   * @default { threshold: 0.05, includeAA: false }
+   * @default { threshold: 0.1, includeAA: false }
    */
-  diffOptions: DiffOptions;
+  diffOptions: PixelmatchOptions;
   /**
-   *
+   * Define odiff diff options
+   * @default { threshold: 0.1, antialiasing: true }
    */
-  odiffOptions: ODiffOptions; // TODO Update description
+  odiffOptions: ODiffOptions;
   /**
    * Browser capabilities
    * @default { chrome: true }
@@ -255,20 +260,17 @@ export interface Config {
   /**
    * Creevey has two built-in stories providers.
    *
-   * `nodejsStoriesProvider` - The first one is used by default except if CSFv3 is enabled in Storybook.
-   * This provider builds and runs storybook in nodejs env, that allows write interaction tests by using Selenium API.
-   * The downside is it depends from project build specific and slightly increases init time.
-   *
-   * `browserStoriesProvider` - The second one is used by default with CSFv3 storybook feature.
-   * It load stories from storybook which is running in browser, like storyshots or loki do it.
+   * `browserStoriesProvider` - Extracts stories directly from the Storybook UI. It loads stories from storybook which is running in browser, like storyshots or loki do it.
    * The downside of this, you can't use interaction tests in Creevey, unless you use CSFv3.
    * Where you can define `play` method for each story
    *
+   * `hybridStoriesProvider` - Combines stories from Storybook with tests from separate files. This is the default provider used in the configuration.
+   *
    * Usage
    * ``` typescript
-   * import { nodejsStoriesProvider as provider } from 'creevey'
-   * // or
    * import { browserStoriesProvider as provider } from 'creevey'
+   * // or
+   * import { hybridStoriesProvider as provider } from 'creevey'
    *
    * // Creevey config
    * module.exports = {
@@ -336,26 +338,38 @@ export interface StoriesProvider {
 
 export type CreeveyConfig = Partial<Config>;
 
-export interface Options {
-  _: string[];
-  config?: string;
-  port: number;
-  ui: boolean;
-  update: boolean | string;
-  debug: boolean;
-  trace: boolean;
-  browser?: string;
-  /**
-   * @deprecated use {@link Config.reporter} instead
-   */
-  reporter?: string;
-  screenDir?: string;
-  reportDir?: string;
-  gridUrl?: string;
-  storybookUrl?: string;
-  failFast?: boolean;
-  odiff?: boolean;
-}
+export const OptionsSchema = v.object({
+  ui: v.optional(v.boolean()),
+  storybookStart: v.optional(v.union([v.string(), v.boolean()])),
+  config: v.optional(v.string()),
+  debug: v.optional(v.boolean()),
+  port: v.number(),
+  failFast: v.optional(v.boolean()),
+  reportDir: v.optional(v.string()),
+  screenDir: v.optional(v.string()),
+  storybookUrl: v.optional(v.string()),
+  storybookPort: v.optional(v.number()),
+  reporter: v.optional(v.string()),
+  odiff: v.optional(v.boolean()),
+  trace: v.optional(v.boolean()),
+  docker: v.optional(v.boolean()),
+});
+
+export const WorkerOptionsSchema = v.object({
+  browser: v.string(),
+  storybookUrl: v.string(),
+  gridUrl: v.optional(v.string()),
+  config: v.optional(v.string()),
+  debug: v.optional(v.boolean()),
+  trace: v.optional(v.boolean()),
+  reportDir: v.optional(v.string()),
+  screenDir: v.optional(v.string()),
+  odiff: v.optional(v.boolean()),
+  port: v.number(),
+});
+
+export type Options = v.InferOutput<typeof OptionsSchema>;
+export type WorkerOptions = v.InferOutput<typeof WorkerOptionsSchema>;
 
 export type WorkerError = 'browser' | 'test' | 'unknown';
 
@@ -403,10 +417,17 @@ export type TestStatus = 'unknown' | 'pending' | 'running' | 'failed' | 'approve
 
 export interface TestResult {
   status: 'failed' | 'success';
+  retries: number;
   // TODO Remove checks `name == browser` in TestResultsView
   // images?: Partial<{ [name: string]: Images }> | Images;
   images?: Partial<Record<string, Images>>;
   error?: string;
+  // Test metadata for reporting
+  duration?: number;
+  attachments?: string[];
+  sessionId?: string;
+  browserName?: string;
+  workerId?: number;
 }
 
 export class ImagesError extends Error {
@@ -441,8 +462,8 @@ export interface BaseCreeveyTestContext {
    * @internal
    */
   screenshots: { imageName?: string; screenshot: Buffer }[];
-  matchImage: (image: Buffer, imageName?: string) => Promise<void>;
-  matchImages: (images: Record<string, Buffer>) => Promise<void>;
+  matchImage: (image: Buffer | string, imageName?: string) => Promise<void>;
+  matchImages: (images: Record<string, Buffer | string>) => Promise<void>;
 }
 
 export interface CreeveyTestContext extends BaseCreeveyTestContext {
@@ -454,6 +475,8 @@ export interface CreeveyTestContext extends BaseCreeveyTestContext {
 export enum TEST_EVENTS {
   RUN_BEGIN = 'start',
   RUN_END = 'end',
+  SUITE_BEGIN = 'suite',
+  SUITE_END = 'suite end',
   TEST_BEGIN = 'test',
   TEST_END = 'test end',
   TEST_FAIL = 'fail',
@@ -478,22 +501,32 @@ export interface FakeTest {
   title: string;
   fullTitle: () => string;
   titlePath: () => string[];
-  currentRetry: () => number;
+  currentRetry: () => number | undefined;
   retires: () => number;
   slow: () => number;
   duration?: number;
   state?: 'failed' | 'passed';
   // NOTE > duration, > duration / 2, > 0
   speed?: 'slow' | 'medium' | 'fast';
-  err?: unknown;
+  err?: string;
   // NOTE: image files
   attachments?: string[];
+
+  // NOTE: Creevey specific fields
+  creevey: {
+    sessionId: string;
+    browserName: string;
+    workerId: number;
+    willRetry: boolean;
+    images: Partial<Record<string, Partial<Images>>>;
+  };
 }
 
 export interface CreeveyStatus {
   isRunning: boolean;
   tests: Partial<Record<string, TestData>>;
   browsers: string[];
+  isUpdateMode: boolean;
 }
 
 export interface CreeveyUpdate {

package/types/global.d.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-var */
 
+declare var __CREEVEY_ENV__: boolean;
 declare var __CREEVEY_SERVER_HOST__: string;
 declare var __CREEVEY_SERVER_PORT__: number | null;
 declare var __CREEVEY_CLIENT_PORT__: number | null;