creevey 0.10.0-beta.31 → 0.10.0-beta.33

package/docs/storybook.md

@@ -0,0 +1,60 @@
+## Creevey Storybook Parameters
+
+Creevey allows you to customize how stories will be captured. You could define parameters on `global`, `kind` or `story` levels. All these parameters are deeply merged by Storybook for each story.
+
+```ts
+// .storybook/preview.tsx
+export const parameters = {
+  creevey: {
+    // Global parameters are applied to all stories
+    captureElement: '#storybook-root',
+  },
+};
+```
+
+```ts
+// stories/MyModal.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { CreeveyMeta, CreeveyStory } from 'creevey';
+import MyModal from './src/components/MyModal';
+
+const Kind: Meta<typeof MyModal> = {
+  title: 'MyModal',
+  component: MyModal,
+  parameters: {
+    creevey: {
+      // It's possible to add additional delay before capturing screenshot
+      delay: 1000,
+
+      // For capturing the whole browser viewport, you can define `null` instead of css selector.
+      captureElement: null,
+
+      // You can skip some stories from capturing, by defining `skip` option:
+      // skip: { "The reason why story is skipped": { in: 'chrome', stories: 'Loading' } }
+      // - `in` - browser name, regex or array of browser names, which are defined in the Creevey config
+      // - `stories` - story name, regex or array of story names
+      // - `tests` - test name, regex or array of test names
+      // NOTE: If you try to skip stories by story name, the storybook name format will be used
+      // For more info see [storybook-export-vs-name-handling](https://storybook.js.org/docs/formats/component-story-format/#storybook-export-vs-name-handling))
+      skip: {
+        "`MyModal` doesn't support ie11": { in: 'ie11' },
+        'Loading stories are flaky in firefox': { in: 'firefox', stories: 'Loading' },
+        "`MyModal` hovering doesn't work correctly": {
+          in: ['firefox', 'chrome'],
+          tests: /.*hover$/,
+        },
+      },
+    },
+  },
+};
+
+export default Kind;
+
+export const Basic: StoryObj<typeof MyModal> = {
+  creevey: {
+    // Lastly some elements can be ignored in capturing screenshot
+    ignoreElements: ['button', '.local-time'],
+  },
+};
+```

package/docs/tests.md

@@ -1,63 +1,68 @@
-## Write tests
+## Write interactive screenshot tests
 
-By default Creevey generate for each story very simple screenshot test. In most cases it would be enough to test your UI. But you may want to do some interactions and capture one or multiple screenshots with different states of your story. For this case you could write custom tests, like this
+In most cases following Storybook's ideology of [writing stories](https://storybook.js.org/docs/get-started/whats-a-story) is enough to test your UI components. Where each component has a separate stories file with its different states. But sometimes you might have pretty complicated components with a lot of interactions and internal states. In this case, you can write tests for your stories.
 
-```tsx
-import React from 'react';
-import { Story } from '@storybook/react';
-import { CreeveyStory } from 'creevey';
-import MyComponent from './src/components/MyComponent';
+There are two different ways how to write interactive tests with Creevey:
 
-export default { title: 'MyComponent' };
+### Write tests in `*.creevey.ts` files
 
-export const Basic: Story & CreeveyStory = () => <MyComponent />;
-Basic.parameters = {
-  creevey: {
-    captureElement: '#storybook-root',
-    tests: {
-      async click() {
-        await this.browser.actions().click(this.captureElement).perform();
+It's the recommended way to write tests. It allows you to run these tests by Creevey itself and utilize webdriver benefits. The crucial part of it is webdriver action calls are more close to real user interactions and mitigate flakiness and false-negative results. Here is a simple example of how to write tests in `*.creevey.ts` files
 
-        await this.expect(await this.takeScreenshot()).to.matchImage('clicked component');
-      },
-    },
-  },
-};
+```ts
+// stories/MyComponent.creevey.ts
+import { kind, story, test } from 'creevey';
+
+kind('MyComponent', () => {
+  story('Story', ({ setStoryParameters }) => {
+    // It's possible to pass Creevey parameters to story
+    setStoryParameters({
+      captureElement: 'span[data-test-id~="x"]',
+      ignoreElements: [],
+    });
+
+    test('idle', async (context) => {
+      await context.matchImage(await context.takeScreenshot());
+    });
+
+    test('input', async (context) => {
+      await context.webdriver.keyboard.press('Tab');
+      const focus = await context.takeScreenshot();
+      await context.webdriver.keyboard.type('Hello Creevey');
+      const input = await context.takeScreenshot();
+      await context.matchImages({ focus, input });
+    });
+  });
+});
 ```
 
-NOTE: Here you define story parameters with simple test `click`. Where you setup capturing element `#storybook-root` then click on that element and taking screenshot to assert it. `this.browser` allow you to access to native selenium webdriver instance you could check [API here](https://www.selenium.dev/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html).
+In the example above, we used Playwright API to interact with the story. But Creevey also supports Selenium webdriver. And in that case `context.webdriver` will be an instance of Selenium webdriver. Obviously Selenium API is different from Playwright.
 
-You also could write more powerful tests with asserting multiple screenshots
+### Using Storybook's `play` function
 
-```tsx
-import React from 'react';
-import { CSFStory } from 'creevey';
-import MyForm from './src/components/MyForm';
+Storybook allows you to write tests in the story file itself by using [`play` function](https://storybook.js.org/docs/writing-tests/component-testing). It's a good way to write simple tests. But there are couple drawbacks of this approach:
 
-export default { title: 'MyForm' };
+- You can have only one test per story. Which is not a big deal, but sometimes you might not want to have multiple stories with the same markup.
+- Tests are running in browser environment and use https://testing-library.com API under the hood. It's good for unit tests, but might not be suitable for visual regression tests, because testing-library relies on DOM API and not even close to real user interactions. For example, you might have a button that could be visible for user, but it's covered by some other transparent element. With testing-library the button easily accessible and clickable, but the user can't interact with it.
 
-export const Basic: CSFStory<JSX.Element> = () => <MyForm />;
-Basic.story = {
-  parameters: {
-    creevey: {
-      captureElement: '#storybook-root',
-      delay: 1000,
-      tests: {
-        async submit() {
-          const input = await this.browser.findElement({ css: '.my-input' });
+Here is an example of how to write tests using Storybook's `play` function:
 
-          const empty = await this.takeScreenshot();
+```tsx
+// stories/MyComponent.stories.tsx
+import React from 'react';
+import { Meta, StoryObj } from '@storybook/react';
+import { fireEvent, within } from '@storybook/test';
+import MyComponent from './src/components/MyComponent';
 
-          await this.browser.actions().click(input).sendKeys('Hello Creevey').sendKeys(this.keys.ENTER).perform();
+export default {
+  title: 'MyComponent',
+  component: MyComponent,
+};
 
-          const submitted = await this.takeScreenshot();
+export const Basic: StoryObj<typeof MyComponent> = {
+  play: async ({ canvasElement }) => {
+    const slider = await within(canvasElement).findByTestId('slider');
 
-          await this.expect({ empty, submitted }).to.matchImages();
-        },
-      },
-    },
+    await fireEvent.change(slider, { target: { value: 50 } });
   },
 };
 ```
-
-NOTE: In this example I fill some simple form and submit it. Also as you could see, I taking two different screenshots `empty` and `submitted` and assert these in the end.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "creevey",
   "description": "Cross-browser screenshot testing tool for Storybook with fancy UI Runner",
-  "version": "0.10.0-beta.31",
+  "version": "0.10.0-beta.33",
   "type": "commonjs",
   "bin": "dist/cli.js",
   "main": "./dist/index.js",

package/src/client/addon/withCreevey.ts

@@ -322,6 +322,7 @@ export function withCreevey(): ReturnType<typeof makeDecorator> {
   });
 }
 
+// TODO It's not accessible from the outside the package
 export async function capture(options?: CaptureOptions): Promise<void> {
   if (!isTestBrowser) return;
 

package/src/client/web/CreeveyApp.tsx

@@ -137,6 +137,7 @@ export function CreeveyApp({ api, initialState }: CreeveyAppProps): JSX.Element
   }, [handleImageApproveNew, handleGoToNextFailedTest]);
 
   const handleApproveAll = useCallback(() => {
+    // TODO Update handled incorrectly
     api?.approveAll();
   }, [api]);
 

package/src/server/docker.ts

@@ -3,8 +3,8 @@ import Logger from 'loglevel';
 import { Writable } from 'stream';
 import Dockerode, { Container } from 'dockerode';
 import { DockerAuth } from '../types.js';
-import { subscribeOn } from './messages.js';
 import { logger } from './logger.js';
+import { setWorkerContainer } from './worker/context.js';
 
 const docker = new Dockerode();
 
@@ -58,12 +58,13 @@ export async function pullImages(
   }
 }
 
-export async function buildImage(imageName: string, dockerfile: string): Promise<void> {
+export async function buildImage(imageName: string, version: string, dockerfile: string): Promise<void> {
   const images = await docker.listImages({ filters: { label: [`creevey=${imageName}`] } });
 
-  if (images.at(0)) {
+  const containers = await docker.listContainers({ all: true, filters: { label: [`creevey=${imageName}`] } });
+  if (containers.length > 0) {
     await Promise.all(
-      (await docker.listContainers({ all: true, filters: { label: [`creevey=${imageName}`] } })).map(async (info) => {
+      containers.map(async (info) => {
         const container = docker.getContainer(info.Id);
         try {
           await container.remove({ force: true });
@@ -72,6 +73,23 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
         }
       }),
     );
+  }
+
+  const oldImages = images.filter((info) => info.Labels.version !== version);
+  if (oldImages.length > 0) {
+    await Promise.all(
+      oldImages.map(async (info) => {
+        const image = docker.getImage(info.Id);
+        try {
+          await image.remove({ force: true });
+        } catch {
+          /* noop */
+        }
+      }),
+    );
+  }
+
+  if (oldImages.length !== images.length) {
     logger().info(`Image ${imageName} already exists`);
     return;
   }
@@ -91,7 +109,7 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
       // @ts-expect-error Type incompatibility AsyncIterator and AsyncIterableIterator
       pack,
       // TODO Support buildkit decode grpc (version: '2')
-      { t: imageName, labels: { creevey: imageName }, version: '1' },
+      { t: imageName, labels: { creevey: imageName, version }, version: '1' },
       (buildError: Error | null, stream) => {
         if (buildError || !stream) {
           // spinner.error(buildError?.message);
@@ -148,14 +166,7 @@ export async function runImage(
 
   return new Promise((resolve) => {
     hub.once('container', (container: Container) => {
-      // eslint-disable-next-line @typescript-eslint/no-misused-promises
-      subscribeOn('shutdown', async () => {
-        try {
-          await container.remove({ force: true });
-        } catch {
-          /* noop */
-        }
-      });
+      setWorkerContainer(container);
     });
     hub.once(
       'start',

package/src/server/index.ts

@@ -6,13 +6,13 @@ import { resolveCommand } from 'package-manager-detector/commands';
 import { readConfig, defaultBrowser } from './config.js';
 import { Options, Config, BrowserConfigObject, isWorkerMessage } from '../types.js';
 import { logger } from './logger.js';
+import { getStorybookUrl, checkIsStorybookConnected } from './connection.js';
 import { SeleniumWebdriver } from './selenium/webdriver.js';
 import { LOCALHOST_REGEXP } from './webdriver.js';
 import { isInsideDocker, killTree, resolvePlaywrightBrowserType, shutdownWithError } from './utils.js';
 import { sendWorkerMessage, subscribeOn } from './messages.js';
 import { buildImage } from './docker.js';
 import { mkdir, writeFile } from 'fs/promises';
-import { getStorybookUrl, checkIsStorybookConnected } from './connection.js';
 import assert from 'assert';
 
 async function startWebdriverServer(browser: string, config: Config, options: Options): Promise<string | undefined> {
@@ -35,7 +35,7 @@ async function startWebdriverServer(browser: string, config: Config, options: Op
       if (cluster.isPrimary) return undefined;
 
       const { browserName } = config.browsers[browser] as BrowserConfigObject;
-      return `creevey://${resolvePlaywrightBrowserType(browserName)}.playwright`;
+      return `creevey://${resolvePlaywrightBrowserType(browserName)}`;
     }
 
     const {
@@ -48,24 +48,21 @@ async function startWebdriverServer(browser: string, config: Config, options: Op
       const { browserName } = config.browsers[browser] as BrowserConfigObject;
 
       const imageName = `creevey/${browserName}:v${version}`;
-      const host = await startPlaywrightContainer(imageName, options.debug);
+      const host = await startPlaywrightContainer(imageName, browser, config, options.debug);
 
       return host;
     } else {
       const { playwrightDockerFile } = await import('./playwright/docker-file.js');
-      const browsers = [
-        ...new Set(
-          Object.values(config.browsers).map(
-            (c) => [(c as BrowserConfigObject).browserName, (c as BrowserConfigObject).playwrightOptions] as const,
-          ),
-        ),
-      ];
+      const {
+        default: { version: creeveyVersion },
+      } = await import('../../package.json', { with: { type: 'json' } });
+      const browsers = [...new Set(Object.values(config.browsers).map((c) => (c as BrowserConfigObject).browserName))];
       await Promise.all(
-        browsers.map(async ([browserName, launchOptions]) => {
+        browsers.map(async (browserName) => {
           const imageName = `creevey/${browserName}:v${version}`;
-          const dockerfile = playwrightDockerFile(browserName, version, launchOptions);
+          const dockerfile = await playwrightDockerFile(browserName, version);
 
-          await buildImage(imageName, dockerfile);
+          await buildImage(imageName, creeveyVersion, dockerfile);
         }),
       );
 
@@ -107,7 +104,7 @@ export default async function (options: Options): Promise<void> {
     gridUrl = await startWebdriverServer(browser, config, options);
   }
 
-  if (cluster.isPrimary) {
+  if (cluster.isPrimary && process.env.CI !== 'true') {
     const [localUrl, remoteUrl] = getStorybookUrl(config, options);
     const pm = getUserAgent();
     assert(pm, new Error('Failed to detect current package manager'));

package/src/server/playwright/docker-file.ts

@@ -1,10 +1,12 @@
+import { readFile } from 'fs/promises';
+import { pathToFileURL } from 'url';
 import semver from 'semver';
 import { exec } from 'shelljs';
-import { LaunchOptions } from 'playwright-core';
-import { resolvePlaywrightBrowserType } from '../utils';
+
+const importMetaUrl = pathToFileURL(__filename).href;
 
 // TODO Support custom docker images
-export function playwrightDockerFile(browser: string, version: string, serverOptions?: LaunchOptions): string {
+export async function playwrightDockerFile(browser: string, version: string): Promise<string> {
   const sv = semver.coerce(version);
 
   let npmRegistry;
@@ -14,14 +16,19 @@ export function playwrightDockerFile(browser: string, version: string, serverOpt
     /* noop */
   }
 
+  const indexJs = await readFile(new URL('./index-source.mjs', importMetaUrl), 'utf-8');
+
   return `
 FROM node:lts
 
 WORKDIR /creevey
 
 RUN echo "{ \\"type\\": \\"module\\" }" > package.json && \\
-    echo "import { ${resolvePlaywrightBrowserType(browser)} as browser } from 'playwright-core';" >> index.js && \\
-    echo "const ws = await browser.launchServer({ ...${JSON.stringify(serverOptions)}, port: 4444, wsPath: 'creevey' })" >> index.js && \\${
+    ${indexJs
+      .split('\n')
+      .map((line) => `echo "${line.replace(/"/g, '\\"')}" >> index.js && \\`)
+      .join('\n')}
+    ${
       npmRegistry
         ? `
     echo "registry=${npmRegistry}" > .npmrc && \\`

package/src/server/playwright/docker.ts

@@ -1,9 +1,17 @@
+import assert from 'assert';
 import { runImage } from '../docker';
 import { emitWorkerMessage, subscribeOn } from '../messages';
-import { isInsideDocker } from '../utils';
+import { getCreeveyCache, isInsideDocker, resolvePlaywrightBrowserType } from '../utils';
 import { LOCALHOST_REGEXP } from '../webdriver';
+import type { BrowserConfigObject, Config } from '../../types';
 
-export async function startPlaywrightContainer(imageName: string, debug: boolean): Promise<string> {
+export async function startPlaywrightContainer(
+  imageName: string,
+  browser: string,
+  config: Config,
+  debug: boolean,
+): Promise<string> {
+  const { browserName, playwrightOptions } = config.browsers[browser] as BrowserConfigObject;
   const port = await new Promise<number>((resolve) => {
     subscribeOn('worker', (message) => {
       if (message.type == 'port') {
@@ -13,13 +21,18 @@ export async function startPlaywrightContainer(imageName: string, debug: boolean
     emitWorkerMessage({ type: 'port', payload: { port: -1 } });
   });
 
+  const cacheDir = await getCreeveyCache();
+
+  assert(cacheDir, "Couldn't get cache directory");
+
   const host = await runImage(
     imageName,
-    [],
+    [JSON.stringify({ ...playwrightOptions, browser: resolvePlaywrightBrowserType(browserName) })],
     {
       ExposedPorts: { [`${port}/tcp`]: {} },
       HostConfig: {
         PortBindings: { ['4444/tcp']: [{ HostPort: `${port}` }] },
+        Binds: [`${cacheDir}/${process.pid}:/creevey/traces`],
       },
     },
     debug,

package/src/server/playwright/index-source.mjs

@@ -0,0 +1,16 @@
+import { chromium, firefox, webkit } from 'playwright-core';
+
+/** @type import("playwright-core").LaunchOptions & { browser: 'chromium' | 'firefox' | 'webkit' } */
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const config = JSON.parse(process.argv.slice(2)[0]);
+
+const browsers = { chromium, firefox, webkit };
+
+const ws = await browsers[config.browser].launchServer({
+  ...config,
+  port: 4444,
+  wsPath: 'creevey',
+  tracesDir: 'traces',
+});
+
+console.log(config.browser, 'browser server launched on:', ws.wsEndpoint());