creevey 0.10.0-beta.4 → 0.10.0-beta.41

package/docs/config.md

@@ -1,212 +1,226 @@
-## Config/Options
+# Creevey Configuration Examples
 
-### CLI Options
+Creevey is highly configurable and can be tailored to your specific needs. Here are some examples of how to configure Creevey for your Storybook.
 
-- `--config` — Specify path to config file. Default `.creevey/config.js` or `creevey.config.js`
-- `--ui` — Start runner web server
-- `--update` — Approve all images from `report` directory
-- `--port` — Specify port for web server. Default `3000`
-- `--reportDir` — Path where reports will be stored
-- `--screenDir` — Path where reference images are located
-- `--debug` — Enable debug output
+## Zero Configuration
 
-### Creevey config
+The simplest way to get started with Creevey, just run it without any configuration:
 
-In default configuration Creevey take screenshots of `#storybook-root` element only for chrome browser in one concurrent instance, to run tests in different browsers or speedup tests and run in parallel, you need to define config file `.creevey/config.js`
+```bash
+yarn creevey -s
+```
+
+This will start Creevey with default settings, using Chrome as the browser and also start Storybook hosted at `http://localhost:6006`. Adding `--ui` flag will start the web server for the UI Runner.
+
+**NOTE** By default, Creevey uses Selenium WebDriver, but it's done for backward compatibility and it's recommended to define webdriver explicitly in the configuration.
 
 ```ts
-module.exports = {
-  browsers: {
-    chrome: {
-      browserName: 'chrome',
-      // Define initial viewport size
-      viewport: { width: 1024, height: 720 },
-      // Increase parallel sessions
-      limit: 2,
-    },
-    firefox: {
-      browserName: 'firefox',
-      viewport: { width: 1024, height: 720 },
-    },
-  },
+// creevey.config.ts
+import { SeleniumWebdriver } from 'creevey/selenium';
+
+const config = {
+  webdriver: SeleniumWebdriver,
+};
+
+// or use Playwright instead
+
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config = {
+  webdriver: PlaywrightWebdriver,
 };
 ```
 
-### All possible config options
+## Basic Configuration
 
-:warning: **WARN** :warning: This config is just an example with all possible options. :warning: **WARN** :warning:
+The minimal configuration to test your stories might be:
 
 ```ts
-const path = require('path');
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
 
-module.exports = {
-  // Specify custom Selenium Grid url (see usage below)
-  // In most cases you don't need this option
-  gridUrl: '<gridUrl>/wd/hub',
+  // The URL where your Storybook is hosted
+  storybookUrl: 'http://localhost:9000',
+
+  // Define which browsers to use for testing
+  browsers: {
+    chromium: {
+      // The browser type name which will be used by Playwright
+      browserName: 'chromium',
+      // Default viewport dimensions
+      viewport: { width: 1024, height: 768 },
+      // Limit of retries for failed tests
+      maxRetries: 2,
+    },
+  },
+};
 
-  // Default Storybook url
-  storybookUrl: 'http://localhost:6006',
+export default config;
+```
 
-  // Where original images are stored
-  screenDir: path.join(__dirname, '../images'),
+## Multiple Browsers Configuration
 
-  // Report directory that contains data from previous runs
-  reportDir: path.join(__dirname, '../report'),
+It's possible to est your stories across different browsers:
 
-  // Pixelmatch options
-  diffOptions: { threshold: 0.1 },
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
 
-  // How many times test should be retried before to consider it as failed
-  maxRetries: 2,
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
 
-  // Describe browsers and their options
   browsers: {
-    // Shorthand declarations of browsers
-    chrome: true,
-    ff: 'firefox',
-
-    otherChrome: {
-      browserName: 'chrome',
-      // Define initial viewport size
-      viewport: { width: 1024, height: 720 },
-      // Increase parallel sessions
+    chromium: {
+      browserName: 'chromium',
+      // Amount of parallel browser sessions
       limit: 2,
-      /* Also you can define any browser capabilities here */
-      version: '86.0',
-      // It's possible to set Storybook's globals
-      // https://github.com/storybookjs/storybook/blob/v6.0.0/docs/essentials/toolbars-and-globals.md
-      // NOTE: This is an experimental feature and will be replaced in future
-      _storybookGlobals: {
-        myTheme: 'dark',
-      },
     },
-
-    // You can override some global options for specific browser
-    ie11: {
-      browserName: 'internet explorer',
-      // Like user another Selenium Grid url
-      gridUrl: '<anotherGridUrl>/wd/hub',
-      // Or use different storybook instance
-      storybookUrl: 'http://mystoryhost:6007',
-      // And use you own docker image
-      // By default Creevey will use selenoid image according browser name and version:
-      // `selenoid/${browserName}:${version ?? 'latest'}` image
-      dockerImage: 'microsoft/ie:11.0',
+    firefox: {
+      browserName: 'firefox',
+      limit: 2,
     },
   },
+};
+
+export default config;
+```
+
+## Test stories with different themes
+
+Test your components with different themes:
 
-  // You may want to do something before tests started (for example start browserstack-local)
-  hooks: {
-    async before() {
-      /* ... */
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+  browsers: {
+    dark: {
+      browserName: 'chromium',
+      // Define a storybook globals which will be applied to the stories
+      _storybookGlobals: {
+        theme: 'dark',
+      },
     },
-    async after() {
-      /* ... */
+    light: {
+      browserName: 'chromium',
+      _storybookGlobals: {
+        theme: 'light',
+      },
     },
   },
 };
+
+export default config;
 ```
 
-### Storybook parameters
+## Using dedicated Selenium Grid server
 
-Also you could define parameters on `global`, `kind` or `story` levels. All these parameters are deeply merged by Storybook for each story. But bear in mind when you define skip option as an array Storybook treats it as primitive value and doesn't merge with other skip options.
+Use Selenium Grid for distributed testing:
 
-```tsx
-// .storybook/preview.tsx
-export const parameters = {
-  creevey: {
-    // Skip all *hover tests in IE11 on the global level
-    skip: {
-      "hovers don't work in ie11": { in: 'ie11', tests: /.*hover$/ },
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { SeleniumWebdriver } from 'creevey/selenium';
+
+const config: CreeveyConfig = {
+  webdriver: SeleniumWebdriver,
+  // Selenium Grid connection settings
+  gridUrl: 'http://selenium-hub:4444/wd/hub',
+  browsers: {
+    chrome: {
+      browserName: 'chrome',
+      // You can define any additional selenium capabilities here
+      // https://w3c.github.io/webdriver/#capabilities
+      seleniumCapabilities: {
+        browserVersion: '128.0',
+        platformName: 'linux',
+      },
     },
   },
 };
+
+export default config;
 ```
 
-```tsx
-import React from 'react';
-import { Meta, Story } from '@storybook/react';
-import { CreeveyMeta, CreeveyStory } from 'creevey';
-import MyComponent from './src/components/MyComponent';
-
-export default {
-  title: 'MyComponent'
-  parameters: {
-    creevey: {
-      // You could skip some browsers/stories or even specific tests
-      skip: {
-        "`MyComponent` doesn't support ie11": { in: 'ie11' },
-        "Loading stories are flaky in firefox": { in: 'firefox', stories: 'Loading' },
-        "`MyComponent` hovering doesn't work correctly": {
-          in: ['firefox', 'chrome'],
-          tests: /.*hover$/,
-        },
+## Playwright configuration options
+
+Leverage Playwright for testing:
+
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+  browsers: {
+    chromium: {
+      browserName: 'chromium',
+      // Playwright-specific options
+      // https://playwright.dev/docs/api/class-browsertype#browser-type-launch-server
+      playwrightOptions: {
+        headless: false,
+        channel: 'chrome-canary',
+        slowMo: 50, // Slow down Playwright operations by 50ms
       },
     },
   },
-} as Meta & CreeveyMeta;
-
-export const Basic: Story & CreeveyStory = () => <MyComponent />;
-Basic.parameters = {
-  creevey: {
-    captureElement: '.container',
-    // elements to ignore in capturing screenshot
-    ignoreElements: ['button', '.local-time'],
-    // Delay before test starts in ms
-    delay: 1000,
-    tests: {
-      /* ... */
-    },
-  },
 };
+
+export default config;
 ```
 
-### `skip` option examples:
+## Advanced Configuration
+
+Comprehensive example combining multiple features and additional options:
 
 ```ts
-interface SkipOption {
-  in?: string | string[] | RegExp;
-  kinds?: string | string[] | RegExp;
-  stories?: string | string[] | RegExp;
-  tests?: string | string[] | RegExp;
-}
-
-type SkipOptions = boolean | string | Record<string, SkipOption | SkipOption[]>;
-```
+// creevey.config.ts
+import path from 'path';
+import MochaJUnitReporter from 'mocha-junit-reporter';
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+
+  // It's possible to resolve Storybook URL at the runtime
+  resolveStorybookUrl: () =>
+    fetch('https://example.com/resolve-ip')
+      .then((res) => res.text())
+      .then((data) => `http://${data}:6006`),
+
+  // Define custom reference screenshots directory
+  screenDir: path.join(process.cwd(), 'screenshots'),
+
+  // Define custom report directory
+  reportDir: path.join(process.cwd(), 'reports'),
+
+  // Define path where Creevey tests are located
+  testDir: path.join(process.cwd(), 'tests'),
+
+  // You can use any Mocha-like reporter
+  reporter: MochaJUnitReporter,
 
-- Skip all stories for all browsers:
-  - `skip: true`
-  - `skip: 'Skip reason message'`
-  - `skip: { 'Skip reason message': true }`
-- Skip all stories for specific browsers:
-  - `skip: { 'Skip reason message': { in: 'ie11' } }`
-  - `skip: { 'Skip reason message': { in: ['ie11', 'chrome'] } }`
-  - `skip: { 'Skip reason message': { in: /^fire.*/ } }`
-- Skip all stories in specific kinds:
-  - `skip: { 'Skip reason message': { kinds: 'Button' } }`
-  - `skip: { 'Skip reason message': { kinds: ['Button', 'Input'] } }`
-  - `skip: { 'Skip reason message': { kinds: /.*Modal$/ } }`
-- Skip all tests in specific stories:
-  - `skip: { 'Skip reason message': { stories: 'simple' } }`
-  - `skip: { 'Skip reason message': { stories: ['simple', 'special'] } }`
-  - `skip: { 'Skip reason message': { stories: /.*large$/ } }`
-- Skip specific tests:
-  - `skip: { 'Skip reason message': { tests: 'click' } }`
-  - `skip: { 'Skip reason message': { tests: ['hover', 'click'] } }`
-  - `skip: { 'Skip reason message': { tests: /^press.*$/ } }`
-- Multiple skip options:
-  - for one reason
-    ```
-    skip: {
-      "reason": [{ /* ... */ }, { /* ... */ }],
-    }
-    ```
-  - for several reasons
-    ```
-    skip: {
-      "reason 1": { /* ... */ },
-      "reason 2": { /* ... */ },
-    }
-    ```
-
-NOTE: If you try to skip stories by story name, the storybook name format will be used (For more info see [storybook-export-vs-name-handling](https://storybook.js.org/docs/formats/component-story-format/#storybook-export-vs-name-handling))
+  // Pixelmatch options
+  diffOptions: { threshold: 0.1 },
+
+  // Regex pattern to match test files
+  testsRegex: /\.creevey\.ts$/,
+
+  // Disable using docker, to start browsers locally, it's useful for CI
+  useDocker: process.env.CI,
+
+  browsers: {
+    /* ... */
+  },
+};
+```

package/docs/storybook.md

@@ -0,0 +1,60 @@
+## Creevey Storybook Parameters
+
+Creevey allows you to customize how stories will be captured. You could define parameters on `global`, `kind` or `story` levels. All these parameters are deeply merged by Storybook for each story.
+
+```ts
+// .storybook/preview.tsx
+export const parameters = {
+  creevey: {
+    // Global parameters are applied to all stories
+    captureElement: '#storybook-root',
+  },
+};
+```
+
+```ts
+// stories/MyModal.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { CreeveyMeta, CreeveyStory } from 'creevey';
+import MyModal from './src/components/MyModal';
+
+const Kind: Meta<typeof MyModal> = {
+  title: 'MyModal',
+  component: MyModal,
+  parameters: {
+    creevey: {
+      // It's possible to add additional delay before capturing screenshot
+      delay: 1000,
+
+      // For capturing the whole browser viewport, you can define `null` instead of css selector.
+      captureElement: null,
+
+      // You can skip some stories from capturing, by defining `skip` option:
+      // skip: { "The reason why story is skipped": { in: 'chrome', stories: 'Loading' } }
+      // - `in` - browser name, regex or array of browser names, which are defined in the Creevey config
+      // - `stories` - story name, regex or array of story names
+      // - `tests` - test name, regex or array of test names
+      // NOTE: If you try to skip stories by story name, the storybook name format will be used
+      // For more info see [storybook-export-vs-name-handling](https://storybook.js.org/docs/formats/component-story-format/#storybook-export-vs-name-handling))
+      skip: {
+        "`MyModal` doesn't support ie11": { in: 'ie11' },
+        'Loading stories are flaky in firefox': { in: 'firefox', stories: 'Loading' },
+        "`MyModal` hovering doesn't work correctly": {
+          in: ['firefox', 'chrome'],
+          tests: /.*hover$/,
+        },
+      },
+    },
+  },
+};
+
+export default Kind;
+
+export const Basic: StoryObj<typeof MyModal> = {
+  creevey: {
+    // Lastly some elements can be ignored in capturing screenshot
+    ignoreElements: ['button', '.local-time'],
+  },
+};
+```

package/docs/tests.md

@@ -1,63 +1,68 @@
-## Write tests
+## Write interactive screenshot tests
 
-By default Creevey generate for each story very simple screenshot test. In most cases it would be enough to test your UI. But you may want to do some interactions and capture one or multiple screenshots with different states of your story. For this case you could write custom tests, like this
+In most cases following Storybook's ideology of [writing stories](https://storybook.js.org/docs/get-started/whats-a-story) is enough to test your UI components. Where each component has a separate stories file with its different states. But sometimes you might have pretty complicated components with a lot of interactions and internal states. In this case, you can write tests for your stories.
 
-```tsx
-import React from 'react';
-import { Story } from '@storybook/react';
-import { CreeveyStory } from 'creevey';
-import MyComponent from './src/components/MyComponent';
+There are two different ways how to write interactive tests with Creevey:
 
-export default { title: 'MyComponent' };
+### Write tests in `*.creevey.ts` files
 
-export const Basic: Story & CreeveyStory = () => <MyComponent />;
-Basic.parameters = {
-  creevey: {
-    captureElement: '#storybook-root',
-    tests: {
-      async click() {
-        await this.browser.actions().click(this.captureElement).perform();
+It's the recommended way to write tests. It allows you to run these tests by Creevey itself and utilize webdriver benefits. The crucial part of it is webdriver action calls are more close to real user interactions and mitigate flakiness and false-negative results. Here is a simple example of how to write tests in `*.creevey.ts` files
 
-        await this.expect(await this.takeScreenshot()).to.matchImage('clicked component');
-      },
-    },
-  },
-};
+```ts
+// stories/MyComponent.creevey.ts
+import { kind, story, test } from 'creevey';
+
+kind('MyComponent', () => {
+  story('Story', ({ setStoryParameters }) => {
+    // It's possible to pass Creevey parameters to story
+    setStoryParameters({
+      captureElement: 'span[data-test-id~="x"]',
+      ignoreElements: [],
+    });
+
+    test('idle', async (context) => {
+      await context.matchImage(await context.takeScreenshot());
+    });
+
+    test('input', async (context) => {
+      await context.webdriver.keyboard.press('Tab');
+      const focus = await context.takeScreenshot();
+      await context.webdriver.keyboard.type('Hello Creevey');
+      const input = await context.takeScreenshot();
+      await context.matchImages({ focus, input });
+    });
+  });
+});
 ```
 
-NOTE: Here you define story parameters with simple test `click`. Where you setup capturing element `#storybook-root` then click on that element and taking screenshot to assert it. `this.browser` allow you to access to native selenium webdriver instance you could check [API here](https://www.selenium.dev/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html).
+In the example above, we used Playwright API to interact with the story. But Creevey also supports Selenium webdriver. And in that case `context.webdriver` will be an instance of Selenium webdriver. Obviously Selenium API is different from Playwright.
 
-You also could write more powerful tests with asserting multiple screenshots
+### Using Storybook's `play` function
 
-```tsx
-import React from 'react';
-import { CSFStory } from 'creevey';
-import MyForm from './src/components/MyForm';
+Storybook allows you to write tests in the story file itself by using [`play` function](https://storybook.js.org/docs/writing-tests/component-testing). It's a good way to write simple tests. But there are couple drawbacks of this approach:
 
-export default { title: 'MyForm' };
+- You can have only one test per story. Which is not a big deal, but sometimes you might not want to have multiple stories with the same markup.
+- Tests are running in browser environment and use https://testing-library.com API under the hood. It's good for unit tests, but might not be suitable for visual regression tests, because testing-library relies on DOM API and not even close to real user interactions. For example, you might have a button that could be visible for user, but it's covered by some other transparent element. With testing-library the button easily accessible and clickable, but the user can't interact with it.
 
-export const Basic: CSFStory<JSX.Element> = () => <MyForm />;
-Basic.story = {
-  parameters: {
-    creevey: {
-      captureElement: '#storybook-root',
-      delay: 1000,
-      tests: {
-        async submit() {
-          const input = await this.browser.findElement({ css: '.my-input' });
+Here is an example of how to write tests using Storybook's `play` function:
 
-          const empty = await this.takeScreenshot();
+```tsx
+// stories/MyComponent.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { fireEvent, within } from '@storybook/test';
+import MyComponent from './src/components/MyComponent';
 
-          await this.browser.actions().click(input).sendKeys('Hello Creevey').sendKeys(this.keys.ENTER).perform();
+export default {
+  title: 'MyComponent',
+  component: MyComponent,
+};
 
-          const submitted = await this.takeScreenshot();
+export const Basic: StoryObj<typeof MyComponent> = {
+  play: async ({ canvasElement }) => {
+    const slider = await within(canvasElement).findByTestId('slider');
 
-          await this.expect({ empty, submitted }).to.matchImages();
-        },
-      },
-    },
+    await fireEvent.change(slider, { target: { value: 50 } });
   },
 };
 ```
-
-NOTE: In this example I fill some simple form and submit it. Also as you could see, I taking two different screenshots `empty` and `submitted` and assert these in the end.