creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/docs/config.md

@@ -1,212 +1,226 @@
-## Config/Options
+# Creevey Configuration Examples
 
-### CLI Options
+Creevey is highly configurable and can be tailored to your specific needs. Here are some examples of how to configure Creevey for your Storybook.
 
-- `--config` — Specify path to config file. Default `.creevey/config.js` or `creevey.config.js`
-- `--ui` — Start runner web server
-- `--update` — Approve all images from `report` directory
-- `--port` — Specify port for web server. Default `3000`
-- `--reportDir` — Path where reports will be stored
-- `--screenDir` — Path where reference images are located
-- `--debug` — Enable debug output
+## Zero Configuration
 
-### Creevey config
+The simplest way to get started with Creevey, just run it without any configuration:
 
-In default configuration Creevey take screenshots of `#storybook-root` element only for chrome browser in one concurrent instance, to run tests in different browsers or speedup tests and run in parallel, you need to define config file `.creevey/config.js`
+```bash
+yarn creevey test -s
+```
+
+This will start Creevey with default settings, using Chrome as the browser and also start Storybook hosted at `http://localhost:6006`. Adding `--ui` flag will start the web server for the UI Runner.
+
+**NOTE** By default, Creevey uses Selenium WebDriver, but it's done for backward compatibility and it's recommended to define webdriver explicitly in the configuration.
 
 ```ts
-module.exports = {
-  browsers: {
-    chrome: {
-      browserName: 'chrome',
-      // Define initial viewport size
-      viewport: { width: 1024, height: 720 },
-      // Increase parallel sessions
-      limit: 2,
-    },
-    firefox: {
-      browserName: 'firefox',
-      viewport: { width: 1024, height: 720 },
-    },
-  },
+// creevey.config.ts
+import { SeleniumWebdriver } from 'creevey/selenium';
+
+const config = {
+  webdriver: SeleniumWebdriver,
+};
+
+// or use Playwright instead
+
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config = {
+  webdriver: PlaywrightWebdriver,
 };
 ```
 
-### All possible config options
+## Basic Configuration
 
-:warning: **WARN** :warning: This config is just an example with all possible options. :warning: **WARN** :warning:
+The minimal configuration to test your stories might be:
 
 ```ts
-const path = require('path');
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
 
-module.exports = {
-  // Specify custom Selenium Grid url (see usage below)
-  // In most cases you don't need this option
-  gridUrl: '<gridUrl>/wd/hub',
+  // The URL where your Storybook is hosted
+  storybookUrl: 'http://localhost:9000',
+
+  // Define which browsers to use for testing
+  browsers: {
+    chromium: {
+      // The browser type name which will be used by Playwright
+      browserName: 'chromium',
+      // Default viewport dimensions
+      viewport: { width: 1024, height: 768 },
+      // Limit of retries for failed tests
+      maxRetries: 2,
+    },
+  },
+};
 
-  // Default Storybook url
-  storybookUrl: 'http://localhost:6006',
+export default config;
+```
 
-  // Where original images are stored
-  screenDir: path.join(__dirname, '../images'),
+## Multiple Browsers Configuration
 
-  // Report directory that contains data from previous runs
-  reportDir: path.join(__dirname, '../report'),
+It's possible to est your stories across different browsers:
 
-  // Pixelmatch options
-  diffOptions: { threshold: 0.1 },
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
 
-  // How many times test should be retried before to consider it as failed
-  maxRetries: 2,
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
 
-  // Describe browsers and their options
   browsers: {
-    // Shorthand declarations of browsers
-    chrome: true,
-    ff: 'firefox',
-
-    otherChrome: {
-      browserName: 'chrome',
-      // Define initial viewport size
-      viewport: { width: 1024, height: 720 },
-      // Increase parallel sessions
+    chromium: {
+      browserName: 'chromium',
+      // Amount of parallel browser sessions
       limit: 2,
-      /* Also you can define any browser capabilities here */
-      version: '86.0',
-      // It's possible to set Storybook's globals
-      // https://github.com/storybookjs/storybook/blob/v6.0.0/docs/essentials/toolbars-and-globals.md
-      // NOTE: This is an experimental feature and will be replaced in future
-      _storybookGlobals: {
-        myTheme: 'dark',
-      },
     },
-
-    // You can override some global options for specific browser
-    ie11: {
-      browserName: 'internet explorer',
-      // Like user another Selenium Grid url
-      gridUrl: '<anotherGridUrl>/wd/hub',
-      // Or use different storybook instance
-      storybookUrl: 'http://mystoryhost:6007',
-      // And use you own docker image
-      // By default Creevey will use selenoid image according browser name and version:
-      // `selenoid/${browserName}:${version ?? 'latest'}` image
-      dockerImage: 'microsoft/ie:11.0',
+    firefox: {
+      browserName: 'firefox',
+      limit: 2,
     },
   },
+};
+
+export default config;
+```
+
+## Test stories with different themes
+
+Test your components with different themes:
 
-  // You may want to do something before tests started (for example start browserstack-local)
-  hooks: {
-    async before() {
-      /* ... */
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+  browsers: {
+    dark: {
+      browserName: 'chromium',
+      // Define a storybook globals which will be applied to the stories
+      _storybookGlobals: {
+        theme: 'dark',
+      },
     },
-    async after() {
-      /* ... */
+    light: {
+      browserName: 'chromium',
+      _storybookGlobals: {
+        theme: 'light',
+      },
     },
   },
 };
+
+export default config;
 ```
 
-### Storybook parameters
+## Using dedicated Selenium Grid server
 
-Also you could define parameters on `global`, `kind` or `story` levels. All these parameters are deeply merged by Storybook for each story. But bear in mind when you define skip option as an array Storybook treats it as primitive value and doesn't merge with other skip options.
+Use Selenium Grid for distributed testing:
 
-```tsx
-// .storybook/preview.tsx
-export const parameters = {
-  creevey: {
-    // Skip all *hover tests in IE11 on the global level
-    skip: {
-      "hovers don't work in ie11": { in: 'ie11', tests: /.*hover$/ },
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { SeleniumWebdriver } from 'creevey/selenium';
+
+const config: CreeveyConfig = {
+  webdriver: SeleniumWebdriver,
+  // Selenium Grid connection settings
+  gridUrl: 'http://selenium-hub:4444/wd/hub',
+  browsers: {
+    chrome: {
+      browserName: 'chrome',
+      // You can define any additional selenium capabilities here
+      // https://w3c.github.io/webdriver/#capabilities
+      seleniumCapabilities: {
+        browserVersion: '128.0',
+        platformName: 'linux',
+      },
     },
   },
 };
+
+export default config;
 ```
 
-```tsx
-import React from 'react';
-import { Meta, Story } from '@storybook/react';
-import { CreeveyMeta, CreeveyStory } from 'creevey';
-import MyComponent from './src/components/MyComponent';
-
-export default {
-  title: 'MyComponent'
-  parameters: {
-    creevey: {
-      // You could skip some browsers/stories or even specific tests
-      skip: {
-        "`MyComponent` doesn't support ie11": { in: 'ie11' },
-        "Loading stories are flaky in firefox": { in: 'firefox', stories: 'Loading' },
-        "`MyComponent` hovering doesn't work correctly": {
-          in: ['firefox', 'chrome'],
-          tests: /.*hover$/,
-        },
+## Playwright configuration options
+
+Leverage Playwright for testing:
+
+```ts
+// creevey.config.ts
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+  browsers: {
+    chromium: {
+      browserName: 'chromium',
+      // Playwright-specific options
+      // https://playwright.dev/docs/api/class-browsertype#browser-type-launch-server
+      playwrightOptions: {
+        headless: false,
+        channel: 'chrome-canary',
+        slowMo: 50, // Slow down Playwright operations by 50ms
       },
     },
   },
-} as Meta & CreeveyMeta;
-
-export const Basic: Story & CreeveyStory = () => <MyComponent />;
-Basic.parameters = {
-  creevey: {
-    captureElement: '.container',
-    // elements to ignore in capturing screenshot
-    ignoreElements: ['button', '.local-time'],
-    // Delay before test starts in ms
-    delay: 1000,
-    tests: {
-      /* ... */
-    },
-  },
 };
+
+export default config;
 ```
 
-### `skip` option examples:
+## Advanced Configuration
+
+Comprehensive example combining multiple features and additional options:
 
 ```ts
-interface SkipOption {
-  in?: string | string[] | RegExp;
-  kinds?: string | string[] | RegExp;
-  stories?: string | string[] | RegExp;
-  tests?: string | string[] | RegExp;
-}
-
-type SkipOptions = boolean | string | Record<string, SkipOption | SkipOption[]>;
-```
+// creevey.config.ts
+import path from 'path';
+import MochaJUnitReporter from 'mocha-junit-reporter';
+import { CreeveyConfig } from 'creevey';
+import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: PlaywrightWebdriver,
+
+  // It's possible to resolve Storybook URL at the runtime
+  resolveStorybookUrl: () =>
+    fetch('https://example.com/resolve-ip')
+      .then((res) => res.text())
+      .then((data) => `http://${data}:6006`),
+
+  // Define custom reference screenshots directory
+  screenDir: path.join(process.cwd(), 'screenshots'),
+
+  // Define custom report directory
+  reportDir: path.join(process.cwd(), 'reports'),
+
+  // Define path where Creevey tests are located
+  testDir: path.join(process.cwd(), 'tests'),
+
+  // You can use any Mocha-like reporter
+  reporter: MochaJUnitReporter,
 
-- Skip all stories for all browsers:
-  - `skip: true`
-  - `skip: 'Skip reason message'`
-  - `skip: { 'Skip reason message': true }`
-- Skip all stories for specific browsers:
-  - `skip: { 'Skip reason message': { in: 'ie11' } }`
-  - `skip: { 'Skip reason message': { in: ['ie11', 'chrome'] } }`
-  - `skip: { 'Skip reason message': { in: /^fire.*/ } }`
-- Skip all stories in specific kinds:
-  - `skip: { 'Skip reason message': { kinds: 'Button' } }`
-  - `skip: { 'Skip reason message': { kinds: ['Button', 'Input'] } }`
-  - `skip: { 'Skip reason message': { kinds: /.*Modal$/ } }`
-- Skip all tests in specific stories:
-  - `skip: { 'Skip reason message': { stories: 'simple' } }`
-  - `skip: { 'Skip reason message': { stories: ['simple', 'special'] } }`
-  - `skip: { 'Skip reason message': { stories: /.*large$/ } }`
-- Skip specific tests:
-  - `skip: { 'Skip reason message': { tests: 'click' } }`
-  - `skip: { 'Skip reason message': { tests: ['hover', 'click'] } }`
-  - `skip: { 'Skip reason message': { tests: /^press.*$/ } }`
-- Multiple skip options:
-  - for one reason
-    ```
-    skip: {
-      "reason": [{ /* ... */ }, { /* ... */ }],
-    }
-    ```
-  - for several reasons
-    ```
-    skip: {
-      "reason 1": { /* ... */ },
-      "reason 2": { /* ... */ },
-    }
-    ```
-
-NOTE: If you try to skip stories by story name, the storybook name format will be used (For more info see [storybook-export-vs-name-handling](https://storybook.js.org/docs/formats/component-story-format/#storybook-export-vs-name-handling))
+  // Pixelmatch options
+  diffOptions: { threshold: 0.1 },
+
+  // Regex pattern to match test files
+  testsRegex: /\.creevey\.ts$/,
+
+  // Disable using docker, to start browsers locally, it's useful for CI
+  useDocker: process.env.CI,
+
+  browsers: {
+    /* ... */
+  },
+};
+```

package/docs/examples/playwright-reporer/playwright.config.ts

@@ -0,0 +1,37 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  testMatch: 'visual.spec.ts',
+  globalSetup: 'creevey/playwright/setup',
+  // Configure reporters
+  reporter: [
+    ['list'],
+    // Creevey reporter with debug enabled when playwright is run with `--debug` flag
+    ['creevey/playwright/reporter', { debug: !!process.env.PWDEBUG }],
+  ],
+  // Reference images and report will be saved in these directories
+  snapshotDir: './images',
+  outputDir: './report',
+  use: {
+    viewport: { width: 1280, height: 720 },
+    baseURL: 'http://localhost:6006', 
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+      metadata: {
+        // Define storybook globals here
+        storybookGlobals: {
+          theme: 'dark',
+        },
+      },
+    }
+  ],
+  webServer: {
+    command: 'yarn storybook dev --ci -p 6006',
+    url: 'http://localhost:6006',
+    reuseExistingServer: !process.env.CI,
+  },
+});

package/docs/migration-0.9-to-0.10.md

@@ -0,0 +1,144 @@
+# Migrating from Creevey 0.9 to 0.10
+
+This guide outlines the key changes and steps required to update your Creevey setup from version 0.9 to 0.10. Version 0.10 introduces several significant updates, including Playwright support, a new test context API, and changes to configuration.
+
+---
+
+## Breaking Changes
+
+### 1. Browser Configuration in `creevey.config.ts`
+
+The way browsers are configured has been updated to better distinguish between Selenium and Playwright settings.
+
+- The new `webdriver` field lets you choose your WebDriver implementation. While it defaults to Selenium for now, this may change. We recommend explicitly setting it by importing and using either `SeleniumWebdriver` or `PlaywrightWebdriver`.
+- Selenium-specific capabilities (e.g., `browserVersion`, `platformName`) must now be nested under a `seleniumCapabilities` object.
+- Playwright-specific options should be placed under a `playwrightOptions` object.
+
+**Before (0.9):**
+
+```javascript
+// creevey.config.js
+export default {
+  browsers: {
+    chrome: {
+      browserName: 'chrome',
+      browserVersion: '90.0',
+      platformName: 'linux',
+    },
+  },
+};
+```
+
+**After (0.10):**
+
+```typescript
+// creevey.config.ts
+import type { CreeveyConfig } from 'creevey';
+import { SeleniumWebdriver } from 'creevey/selenium';
+// or
+// import { PlaywrightWebdriver } from 'creevey/playwright';
+
+const config: CreeveyConfig = {
+  webdriver: SeleniumWebdriver, // or PlaywrightWebdriver
+  browsers: {
+    chrome: {
+      // For Selenium
+      browserName: 'chrome',
+      seleniumCapabilities: {
+        browserVersion: '90.0',
+        platformName: 'linux',
+      },
+    },
+    firefoxPlaywright: {
+      // For Playwright
+      browserName: 'firefox', // 'chromium', 'firefox', or 'webkit' for Playwright
+      playwrightOptions: {
+        headless: true,
+      },
+    },
+  },
+};
+
+export default config;
+```
+
+### 2. Mocha Removal and New `CreeveyTestContext` API
+
+Creevey no longer uses the Mocha testing framework. Tests now use a `CreeveyTestContext` object passed as an argument to the test function.
+
+- Replace `this` with the `context` parameter in your tests.
+- Image matching methods (e.g., `matchImage`, `takeScreenshot`) are now called on the `context` object.
+- The `context.webdriver` property gives you direct access to the configured WebDriver instance (Selenium or Playwright) for advanced browser interactions.
+
+**Before (0.9 - Mocha style):**
+
+```javascript
+it('should match the image', async function () {
+  this.expect(await this.takeScreenshot()).to.matchImage('example');
+});
+```
+
+**After (0.10 - New context style):**
+
+```javascript
+it('should match the image', async (context) => {
+  await context.matchImage(await context.takeScreenshot(), 'example');
+});
+
+// Example using context.webdriver with Selenium WebDriver API
+it('should interact with an element using Selenium', async (context) => {
+  const seleniumWebDriver = context.webdriver; // Assuming SeleniumWebdriver is configured
+  const element = await seleniumWebDriver.findElement({ css: '#myElement' });
+  await element.click();
+  // ... more Selenium interactions
+  await context.matchImage(await context.takeScreenshot(), 'selenium-interaction');
+});
+
+// Example using context.webdriver with Playwright API
+it('should interact with an element using Playwright', async (context) => {
+  const playwrightPage = context.webdriver; // Assuming PlaywrightWebdriver is configured
+  await playwrightPage.click('#myElement');
+  // ... more Playwright interactions
+  await context.matchImage(await context.takeScreenshot(), 'playwright-interaction');
+});
+```
+
+### 3. Reporter Configuration
+
+Reporter setup has moved from command-line options to the `creevey.config.ts` file.
+
+**Example (0.10):**
+
+```typescript
+// creevey.config.ts
+import type { CreeveyConfig } from 'creevey';
+
+const config: CreeveyConfig = {
+  reporter: 'junit', // or 'teamcity', 'creevey'
+  reporterOptions: {
+    // For JUnit: { outputFile: 'report.xml' }
+    // For Creevey HTML reporter (default): { reportDir: './report' }
+  },
+};
+
+export default config;
+```
+
+---
+
+## Key New Features & Updates
+
+- **Creevey Playwright Reporter:** Integrate Creevey's visual testing into your existing Playwright test suites. See `docs/playwright-reporter.md` for details.
+- **Approve Tests from Report UI (Update Mode):** Approve visual changes directly in the Creevey report UI. Run with `creevey report` to use the web UI for reviewing and approving screenshots.
+- **ODiff Image Comparison:** Use `odiff` for image comparison with the `--odiff` flag and `odiffOptions` in config.
+- **Easier Storybook Autostart:** Use `creevey test -s` or `creevey test --storybook-start` to automatically start Storybook. Creevey will find a free port if necessary.
+
+---
+
+## Other Notable Changes
+
+- **`--no-docker` Option:** A command-line flag to disable Docker, overriding `useDocker: true` in your config.
+
+---
+
+We recommend reviewing the full announcement for details on all new features and improvements. After updating, test your suite thoroughly to ensure a smooth transition.