creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/docs/playwright-reporter.md

@@ -0,0 +1,357 @@
+# Playwright Reporter for Creevey
+
+Creevey provides a Playwright-compatible reporter that allows you to use Creevey's image comparison, reporting, and UI features with Playwright tests.
+
+## Installation
+
+The reporter is included with the Creevey package. Install Playwright and Creevey in your project:
+
+```bash
+npm install --save-dev @playwright/test creevey
+```
+
+## Configuration
+
+Add the Creevey reporter to your Playwright configuration file:
+
+```typescript
+// playwright.config.ts
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: [
+    ['list'], // Standard Playwright reporter for console output
+    [
+      'creevey/playwright-reporter',
+      {
+        // Optional configuration options
+        reportDir: './report', // Directory for report output
+        screenDir: './images', // Directory for reference images
+        port: 3000, // Port for Creevey UI server
+        debug: false, // Enable debug logging
+      },
+    ],
+  ],
+  // Other Playwright configuration...
+});
+```
+
+## Usage
+
+### Taking Screenshots for Comparison
+
+The reporter automatically collects screenshots taken during Playwright tests. You can use Playwright's built-in screenshot capabilities:
+
+#### Using `page.screenshot()`
+
+```typescript
+import { test, expect } from '@playwright/test';
+
+test('capture screenshot with page.screenshot', async ({ page }) => {
+  await page.goto('https://example.com');
+
+  // Take a screenshot and attach it to the test result
+  const screenshot = await page.screenshot();
+  await test.info().attach('example-screenshot', {
+    body: screenshot,
+    contentType: 'image/png',
+  });
+});
+```
+
+#### Using Playwright's expect().toHaveScreenshot()
+
+```typescript
+import { test, expect } from '@playwright/test';
+
+test('capture screenshot with toHaveScreenshot', async ({ page }) => {
+  await page.goto('https://example.com');
+
+  // This will automatically be picked up by the Creevey reporter
+  await expect(page).toHaveScreenshot('example-screenshot.png');
+});
+```
+
+### Adding Creevey Metadata
+
+You can add metadata to your tests to better organize them in the Creevey UI:
+
+```typescript
+import { test } from '@playwright/test';
+
+test.describe('Homepage', () => {
+  test('header elements @creevey{"storyPath": ["Website", "HomePage"], "browser": "chromium"}', async ({ page }) => {
+    await page.goto('https://example.com');
+    // Test code...
+    await page.screenshot({ path: 'header.png' });
+  });
+});
+```
+
+The `@creevey` annotation accepts a JSON object with the following properties:
+
+- `storyPath`: Array of strings defining the hierarchy in the Creevey UI
+- `browser`: Browser name (chromium, firefox, webkit)
+- `testName`: Custom test name (defaults to the test title)
+
+### Viewing Results
+
+After running your tests, you can view the results in the Creevey UI:
+
+1. Open your browser and navigate to `http://localhost:3000` (or the port you configured)
+2. Browse test results, view image comparisons, and approve changes
+
+Run your tests with Playwright:
+
+```bash
+npx playwright test
+```
+
+The Creevey server will automatically start during the test run, and you'll see a message with the URL once tests complete.
+
+### Approving Screenshots
+
+You can approve screenshots directly from the Creevey UI by:
+
+1. Selecting the test with the failed comparison
+2. Reviewing the differences between expected and actual images
+3. Clicking the "Approve" button to update the reference image
+
+## Advanced Usage
+
+### Custom Screenshot Names
+
+You can customize screenshot names to better organize your visual tests:
+
+```typescript
+test('product page layout', async ({ page }) => {
+  await page.goto('https://example.com/products/1');
+
+  // Take multiple screenshots in a single test
+  await test.info().attach('product-header', {
+    body: await page.locator('.product-header').screenshot(),
+    contentType: 'image/png',
+  });
+
+  await test.info().attach('product-details', {
+    body: await page.locator('.product-details').screenshot(),
+    contentType: 'image/png',
+  });
+});
+```
+
+### Multiple Screenshots in One Test
+
+You can capture multiple screenshots in a single test:
+
+```typescript
+test('responsive design checks', async ({ page }) => {
+  await page.goto('https://example.com');
+
+  // Mobile view
+  await page.setViewportSize({ width: 375, height: 667 });
+  await test.info().attach('mobile-view', {
+    body: await page.screenshot(),
+    contentType: 'image/png',
+  });
+
+  // Tablet view
+  await page.setViewportSize({ width: 768, height: 1024 });
+  await test.info().attach('tablet-view', {
+    body: await page.screenshot(),
+    contentType: 'image/png',
+  });
+
+  // Desktop view
+  await page.setViewportSize({ width: 1280, height: 800 });
+  await test.info().attach('desktop-view', {
+    body: await page.screenshot(),
+    contentType: 'image/png',
+  });
+});
+```
+
+### Custom Test Names
+
+By default, the reporter will use the Playwright test title as the test name. You can customize how tests are named by setting meta data:
+
+```typescript
+test('home page visual test', async ({ page }) => {
+  test.info().annotations.push({
+    type: 'creevey',
+    description: JSON.stringify({
+      testName: 'custom-test-name',
+      browser: 'chrome', // Optional, to override browser name
+    }),
+  });
+
+  // Test implementation...
+});
+```
+
+### Integration with Existing Creevey Configuration
+
+If you're already using Creevey for other tests, you can share configuration:
+
+```typescript
+import { defineConfig } from '@playwright/test';
+import creeveyConfig from './creevey.config.js';
+
+export default defineConfig({
+  reporter: [
+    ['list'],
+    [
+      'creevey/playwright-reporter',
+      {
+        reportDir: creeveyConfig.reportDir,
+        screenDir: creeveyConfig.screenDir,
+        port: 3001, // Use a different port to avoid conflicts
+      },
+    ],
+  ],
+});
+```
+
+### Test Steps Integration
+
+The reporter supports Playwright's test steps, capturing screenshots at each step:
+
+```typescript
+test('multi-step test', async ({ page }) => {
+  await test.step('Step 1: Navigate to home', async () => {
+    await page.goto('/');
+    // Screenshot captured from this step will be associated with "Step 1"
+    await test.info().attach('step1', {
+      body: await page.screenshot(),
+      contentType: 'image/png',
+    });
+  });
+
+  await test.step('Step 2: Click button', async () => {
+    await page.click('.button');
+    // This screenshot will be associated with "Step 2"
+    await test.info().attach('step2', {
+      body: await page.screenshot(),
+      contentType: 'image/png',
+    });
+  });
+});
+```
+
+### Integration with Existing Creevey Server
+
+If you're already running a Creevey server, you can configure the reporter to use it:
+
+```typescript
+[
+  CreeveyPlaywrightReporter,
+  {
+    useExistingServer: true, // Don't start a new server
+    port: 3000, // Port of the existing server
+  },
+];
+```
+
+### Performance Optimization
+
+For large test suites with many screenshots, you can enable performance optimizations:
+
+```typescript
+[
+  CreeveyPlaywrightReporter,
+  {
+    batchProcessing: true, // Process screenshots in batches
+    maxConcurrency: 4, // Maximum concurrent screenshot processing
+    lazyInit: true, // Initialize components on demand
+  },
+];
+```
+
+## API Reference
+
+### Reporter Options
+
+| Option                    | Type    | Default      | Description                                           |
+| ------------------------- | ------- | ------------ | ----------------------------------------------------- |
+| `reportDir`               | string  | `'./report'` | Directory where Creevey will store reports            |
+| `screenDir`               | string  | `'./images'` | Directory where reference images are stored           |
+| `port`                    | number  | `3000`       | Port number for the Creevey UI server                 |
+| `debug`                   | boolean | `false`      | Enable debug logging                                  |
+| `uiEnabled`               | boolean | `true`       | Enable or disable the UI server                       |
+| `useExistingServer`       | boolean | `false`      | Use an existing Creevey server instance               |
+| `batchProcessing`         | boolean | `false`      | Process screenshots in batches for better performance |
+| `maxConcurrency`          | number  | `2`          | Maximum concurrent screenshot processing operations   |
+| `lazyInit`                | boolean | `false`      | Initialize components on demand for faster startup    |
+| `retainOrphanedImages`    | boolean | `false`      | Keep images not associated with current tests         |
+| `customComparisonOptions` | object  | `undefined`  | Custom options for image comparison                   |
+
+## Troubleshooting
+
+### Common Issues
+
+- **Port conflicts**: If the Creevey server fails to start, it might be due to port conflicts. Change the port in the configuration.
+- **Missing screenshots**: Ensure your tests are attaching screenshots with unique names or using `toHaveScreenshot()`.
+- **Path issues**: If reference images aren't found, check that the `screenDir` path is correct and accessible.
+
+If screenshots are not appearing in the Creevey UI:
+
+1. Ensure your screenshots are being properly attached to test results
+2. Check that the content type is set to 'image/png' when attaching
+3. Enable debug mode in the reporter configuration to see detailed logs
+
+### Server Issues
+
+If the Creevey server fails to start:
+
+1. Check if the configured port is already in use
+2. Ensure the report directory is writable
+3. Look for error messages in the console output
+
+### Debug Mode
+
+Enable debug logging for more detailed information:
+
+```typescript
+[
+  CreeveyPlaywrightReporter,
+  {
+    debug: true,
+    // Other options...
+  },
+];
+```
+
+## Error Handling
+
+The reporter is designed to handle errors gracefully:
+
+- If the reporter fails to initialize, tests will still run and screenshots will be captured when possible
+- If the server can't start, the reporter will continue to capture screenshots but the UI won't be available
+- All errors are logged with appropriate severity levels and won't cause your tests to fail
+
+### Debugging Reporter Issues
+
+For detailed debugging information, enable debug mode and check the logs:
+
+```typescript
+[
+  CreeveyPlaywrightReporter,
+  {
+    debug: true,
+  },
+];
+```
+
+You can also specify a custom log file for the reporter:
+
+```typescript
+[
+  CreeveyPlaywrightReporter,
+  {
+    debug: true,
+    logFile: './creevey-reporter.log',
+  },
+];
+```
+
+For more help, please check the main Creevey documentation or open an issue on GitHub.

package/docs/storybook.md

@@ -0,0 +1,60 @@
+## Creevey Storybook Parameters
+
+Creevey allows you to customize how stories will be captured. You could define parameters on `global`, `kind` or `story` levels. All these parameters are deeply merged by Storybook for each story.
+
+```ts
+// .storybook/preview.tsx
+export const parameters = {
+  creevey: {
+    // Global parameters are applied to all stories
+    captureElement: '#storybook-root',
+  },
+};
+```
+
+```ts
+// stories/MyModal.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { CreeveyMeta, CreeveyStory } from 'creevey';
+import MyModal from './src/components/MyModal';
+
+const Kind: Meta<typeof MyModal> = {
+  title: 'MyModal',
+  component: MyModal,
+  parameters: {
+    creevey: {
+      // It's possible to add additional delay before capturing screenshot
+      delay: 1000,
+
+      // For capturing the whole browser viewport, you can define `null` instead of css selector.
+      captureElement: null,
+
+      // You can skip some stories from capturing, by defining `skip` option:
+      // skip: { "The reason why story is skipped": { in: 'chrome', stories: 'Loading' } }
+      // - `in` - browser name, regex or array of browser names, which are defined in the Creevey config
+      // - `stories` - story name, regex or array of story names
+      // - `tests` - test name, regex or array of test names
+      // NOTE: If you try to skip stories by story name, the storybook name format will be used
+      // For more info see [storybook-export-vs-name-handling](https://storybook.js.org/docs/formats/component-story-format/#storybook-export-vs-name-handling))
+      skip: {
+        "`MyModal` doesn't support ie11": { in: 'ie11' },
+        'Loading stories are flaky in firefox': { in: 'firefox', stories: 'Loading' },
+        "`MyModal` hovering doesn't work correctly": {
+          in: ['firefox', 'chrome'],
+          tests: /.*hover$/,
+        },
+      },
+    },
+  },
+};
+
+export default Kind;
+
+export const Basic: StoryObj<typeof MyModal> = {
+  creevey: {
+    // Lastly some elements can be ignored in capturing screenshot
+    ignoreElements: ['button', '.local-time'],
+  },
+};
+```

package/docs/tests.md

@@ -1,63 +1,68 @@
-## Write tests
+## Write interactive screenshot tests
 
-By default Creevey generate for each story very simple screenshot test. In most cases it would be enough to test your UI. But you may want to do some interactions and capture one or multiple screenshots with different states of your story. For this case you could write custom tests, like this
+In most cases following Storybook's ideology of [writing stories](https://storybook.js.org/docs/get-started/whats-a-story) is enough to test your UI components. Where each component has a separate stories file with its different states. But sometimes you might have pretty complicated components with a lot of interactions and internal states. In this case, you can write tests for your stories.
 
-```tsx
-import React from 'react';
-import { Story } from '@storybook/react';
-import { CreeveyStory } from 'creevey';
-import MyComponent from './src/components/MyComponent';
+There are two different ways how to write interactive tests with Creevey:
 
-export default { title: 'MyComponent' };
+### Write tests in `*.creevey.ts` files
 
-export const Basic: Story & CreeveyStory = () => <MyComponent />;
-Basic.parameters = {
-  creevey: {
-    captureElement: '#storybook-root',
-    tests: {
-      async click() {
-        await this.browser.actions().click(this.captureElement).perform();
+It's the recommended way to write tests. It allows you to run these tests by Creevey itself and utilize webdriver benefits. The crucial part of it is webdriver action calls are more close to real user interactions and mitigate flakiness and false-negative results. Here is a simple example of how to write tests in `*.creevey.ts` files
 
-        await this.expect(await this.takeScreenshot()).to.matchImage('clicked component');
-      },
-    },
-  },
-};
+```ts
+// stories/MyComponent.creevey.ts
+import { kind, story, test } from 'creevey';
+
+kind('MyComponent', () => {
+  story('Story', ({ setStoryParameters }) => {
+    // It's possible to pass Creevey parameters to story
+    setStoryParameters({
+      captureElement: 'span[data-test-id~="x"]',
+      ignoreElements: [],
+    });
+
+    test('idle', async (context) => {
+      await context.matchImage(await context.takeScreenshot());
+    });
+
+    test('input', async (context) => {
+      await context.webdriver.keyboard.press('Tab');
+      const focus = await context.takeScreenshot();
+      await context.webdriver.keyboard.type('Hello Creevey');
+      const input = await context.takeScreenshot();
+      await context.matchImages({ focus, input });
+    });
+  });
+});
 ```
 
-NOTE: Here you define story parameters with simple test `click`. Where you setup capturing element `#storybook-root` then click on that element and taking screenshot to assert it. `this.browser` allow you to access to native selenium webdriver instance you could check [API here](https://www.selenium.dev/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html).
+In the example above, we used Playwright API to interact with the story. But Creevey also supports Selenium webdriver. And in that case `context.webdriver` will be an instance of Selenium webdriver. Obviously Selenium API is different from Playwright.
 
-You also could write more powerful tests with asserting multiple screenshots
+### Using Storybook's `play` function
 
-```tsx
-import React from 'react';
-import { CSFStory } from 'creevey';
-import MyForm from './src/components/MyForm';
+Storybook allows you to write tests in the story file itself by using [`play` function](https://storybook.js.org/docs/writing-tests/component-testing). It's a good way to write simple tests. But there are couple drawbacks of this approach:
 
-export default { title: 'MyForm' };
+- You can have only one test per story. Which is not a big deal, but sometimes you might not want to have multiple stories with the same markup.
+- Tests are running in browser environment and use https://testing-library.com API under the hood. It's good for unit tests, but might not be suitable for visual regression tests, because testing-library relies on DOM API and not even close to real user interactions. For example, you might have a button that could be visible for user, but it's covered by some other transparent element. With testing-library the button easily accessible and clickable, but the user can't interact with it.
 
-export const Basic: CSFStory<JSX.Element> = () => <MyForm />;
-Basic.story = {
-  parameters: {
-    creevey: {
-      captureElement: '#storybook-root',
-      delay: 1000,
-      tests: {
-        async submit() {
-          const input = await this.browser.findElement({ css: '.my-input' });
+Here is an example of how to write tests using Storybook's `play` function:
 
-          const empty = await this.takeScreenshot();
+```tsx
+// stories/MyComponent.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { fireEvent, within } from '@storybook/test';
+import MyComponent from './src/components/MyComponent';
 
-          await this.browser.actions().click(input).sendKeys('Hello Creevey').sendKeys(this.keys.ENTER).perform();
+export default {
+  title: 'MyComponent',
+  component: MyComponent,
+};
 
-          const submitted = await this.takeScreenshot();
+export const Basic: StoryObj<typeof MyComponent> = {
+  play: async ({ canvasElement }) => {
+    const slider = await within(canvasElement).findByTestId('slider');
 
-          await this.expect({ empty, submitted }).to.matchImages();
-        },
-      },
-    },
+    await fireEvent.change(slider, { target: { value: 50 } });
   },
 };
 ```
-
-NOTE: In this example I fill some simple form and submit it. Also as you could see, I taking two different screenshots `empty` and `submitted` and assert these in the end.