creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/src/server/compare.ts

@@ -0,0 +1,260 @@
+import path from 'path';
+import { Stats } from 'fs';
+import assert from 'assert';
+import { PNG } from 'pngjs';
+import type { ODiffOptions } from 'odiff-bin';
+import type { PixelmatchOptions } from 'pixelmatch';
+import { mkdir, readdir, readFile, stat, writeFile } from 'fs/promises';
+import { Images } from '../types';
+
+export interface ImageContext {
+  attachments: string[];
+  testFullPath: string[];
+  images: Partial<Record<string, Images>>;
+}
+
+interface ImagePaths {
+  imageName: string;
+  actualImageName: string;
+  expectImageName: string;
+  diffImageName: string;
+  expectImageDir: string;
+  reportImageDir: string;
+}
+
+async function getStat(filePath: string): Promise<Stats | null> {
+  try {
+    return await stat(filePath);
+  } catch (error) {
+    if (typeof error == 'object' && error && (error as { code?: unknown }).code === 'ENOENT') {
+      return null;
+    }
+    throw error;
+  }
+}
+
+async function getLastImageNumber(imageDir: string, imageName: string): Promise<number> {
+  const actualImagesRegexp = new RegExp(`${imageName}-actual-(\\d+)\\.png`);
+
+  try {
+    return (
+      (await readdir(imageDir))
+        .map((filename) => filename.replace(actualImagesRegexp, '$1'))
+        .map(Number)
+        .filter((x) => !isNaN(x))
+        .sort((a, b) => b - a)[0] ?? 0
+    );
+  } catch (_error) {
+    return 0;
+  }
+}
+
+async function readExpected(expectImageDir: string, imageName: string): Promise<Buffer> {
+  const expected = await readFile(path.join(expectImageDir, `${imageName}.png`));
+
+  return expected;
+}
+
+async function saveImages(imageDir: string, images: { name: string; data: Buffer }[]): Promise<string[]> {
+  const files: string[] = [];
+  await mkdir(imageDir, { recursive: true });
+  for (const { name, data } of images) {
+    const filePath = path.join(imageDir, name);
+    await writeFile(filePath, data);
+    files.push(filePath);
+  }
+  return files;
+}
+
+async function getImagePaths(
+  config: { screenDir: string; reportDir: string },
+  testFullPath: string[],
+  assertImageName?: string,
+): Promise<ImagePaths> {
+  const testPath = [...testFullPath];
+  const imageName = assertImageName ?? testPath.pop();
+
+  assert(typeof imageName === 'string', `Can't get image name from empty test scope`);
+
+  const expectImageDir = path.join(config.screenDir, ...testPath);
+  const reportImageDir = path.join(config.reportDir, ...testPath);
+  const imageNumber = (await getLastImageNumber(reportImageDir, imageName)) + 1;
+  const actualImageName = `${imageName}-actual-${imageNumber}.png`;
+  const expectImageName = `${imageName}-expect-${imageNumber}.png`;
+  const diffImageName = `${imageName}-diff-${imageNumber}.png`;
+
+  return { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir };
+}
+
+async function getExpected(
+  ctx: ImageContext,
+  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
+): Promise<{
+  expected: Buffer | null;
+  onCompare: (actual: Buffer, expect?: Buffer, diff?: Buffer) => Promise<void>;
+}> {
+  const onCompare = async (actual: Buffer, expect?: Buffer, diff?: Buffer): Promise<void> => {
+    const imagesMeta: { name: string; data: Buffer }[] = [];
+    const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
+
+    imagesMeta.push({ name: image.actual, data: actual });
+
+    if (diff && expect) {
+      image.expect = expectImageName;
+      image.diff = diffImageName;
+      imagesMeta.push({ name: image.expect, data: expect });
+      imagesMeta.push({ name: image.diff, data: diff });
+    }
+    ctx.attachments = await saveImages(reportImageDir, imagesMeta);
+  };
+
+  const expectImageStat = await getStat(path.join(expectImageDir, `${imageName}.png`));
+  if (!expectImageStat) return { expected: null, onCompare };
+
+  const expected = await readExpected(expectImageDir, imageName);
+
+  return { expected, onCompare };
+}
+
+async function getOdiffExpected(
+  ctx: ImageContext,
+  actual: Buffer,
+  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
+): Promise<{ actual: string; expect: string; diff: string }> {
+  const expected = await readExpected(expectImageDir, imageName);
+
+  const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
+  image.expect = expectImageName;
+  image.diff = diffImageName;
+
+  const imagesMeta = [
+    { name: image.actual, data: actual },
+    { name: expectImageName, data: expected },
+  ];
+
+  ctx.attachments = await saveImages(reportImageDir, imagesMeta);
+
+  return {
+    actual: path.join(reportImageDir, actualImageName),
+    expect: path.join(reportImageDir, expectImageName),
+    diff: path.join(reportImageDir, diffImageName),
+  };
+}
+
+function normalizeImageSize(image: PNG, width: number, height: number): Buffer {
+  const normalizedImage = Buffer.alloc(4 * width * height);
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const i = (y * width + x) * 4;
+      if (x < image.width && y < image.height) {
+        const j = (y * image.width + x) * 4;
+        normalizedImage[i + 0] = image.data[j + 0];
+        normalizedImage[i + 1] = image.data[j + 1];
+        normalizedImage[i + 2] = image.data[j + 2];
+        normalizedImage[i + 3] = image.data[j + 3];
+      } else {
+        normalizedImage[i + 0] = 0;
+        normalizedImage[i + 1] = 0;
+        normalizedImage[i + 2] = 0;
+        normalizedImage[i + 3] = 0;
+      }
+    }
+  }
+  return normalizedImage;
+}
+
+function hasDiffPixels(diff: Buffer): boolean {
+  for (let i = 0; i < diff.length; i += 4) {
+    if (diff[i + 0] == 255 && diff[i + 1] == 0 && diff[i + 2] == 0 && diff[i + 3] == 255) return true;
+  }
+  return false;
+}
+
+function compareImages(
+  expect: Buffer,
+  actual: Buffer,
+  pixelmatch: typeof import('pixelmatch'),
+  diffOptions: PixelmatchOptions,
+): { isEqual: boolean; diff: Buffer } {
+  const expectImage = PNG.sync.read(expect);
+  const actualImage = PNG.sync.read(actual);
+
+  const width = Math.max(actualImage.width, expectImage.width);
+  const height = Math.max(actualImage.height, expectImage.height);
+
+  const diffImage = new PNG({ width, height });
+
+  let actualImageData = actualImage.data;
+  if (actualImage.width < width || actualImage.height < height) {
+    actualImageData = normalizeImageSize(actualImage, width, height);
+  }
+
+  let expectImageData = expectImage.data;
+  if (expectImage.width < width || expectImage.height < height) {
+    expectImageData = normalizeImageSize(expectImage, width, height);
+  }
+
+  pixelmatch(expectImageData, actualImageData, diffImage.data, width, height, diffOptions);
+
+  return {
+    isEqual: !hasDiffPixels(diffImage.data),
+    diff: PNG.sync.write(diffImage),
+  };
+}
+
+export function getPixelmatchAssert(
+  pixelmatch: typeof import('pixelmatch'),
+  ctx: ImageContext,
+  config: { screenDir: string; reportDir: string; diffOptions: PixelmatchOptions },
+) {
+  return async function assertImagePixelmatch(actual: Buffer, imageName?: string): Promise<string | undefined> {
+    const { expected, onCompare } = await getExpected(ctx, await getImagePaths(config, ctx.testFullPath, imageName));
+
+    if (expected == null) {
+      await onCompare(actual);
+      return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
+    }
+
+    if (actual.equals(expected)) {
+      await onCompare(actual);
+      return;
+    }
+
+    const { isEqual, diff } = compareImages(expected, actual, pixelmatch, config.diffOptions);
+
+    if (isEqual) {
+      await onCompare(actual);
+      return;
+    }
+
+    await onCompare(actual, expected, diff);
+
+    return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
+  };
+}
+
+export function getOdiffAssert(
+  compare: (typeof import('odiff-bin'))['compare'],
+  ctx: ImageContext,
+  config: { screenDir: string; reportDir: string; odiffOptions?: ODiffOptions },
+) {
+  const diffOptions = {
+    ...config.odiffOptions,
+    noFailOnFsErrors: true,
+  };
+  return async function assertImage(image: Buffer, imageName?: string): Promise<string | undefined> {
+    const { actual, expect, diff } = await getOdiffExpected(
+      ctx,
+      image,
+      await getImagePaths(config, ctx.testFullPath, imageName),
+    );
+    const result = await compare(actual, expect, diff, diffOptions);
+    if (!result.match) {
+      if (result.reason == 'file-not-exists') {
+        return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
+      }
+      return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
+    }
+  };
+}

package/src/server/config.ts

@@ -1,34 +1,46 @@
 import fs from 'fs';
 import path from 'path';
+import cluster from 'cluster';
 import { pathToFileURL } from 'url';
-import { loadStories as browserStoriesProvider } from './providers/browser.js';
-import { Config, BrowserConfig, BrowserConfigObject, Options, isDefined } from '../types.js';
+import * as v from 'valibot';
+import { loadStories as hybridStoriesProvider } from './providers/hybrid.js';
+import {
+  Config,
+  BrowserConfig,
+  BrowserConfigObject,
+  Options,
+  isDefined,
+  WorkerOptions,
+  OptionsSchema,
+} from '../types.js';
 import { configExt, loadThroughTSX } from './utils.js';
-import { CreeveyReporter, TeamcityReporter } from './reporter.js';
+import { CreeveyReporter } from './reporters/creevey.js';
+import { TeamcityReporter } from './reporters/teamcity.js';
 import { logger } from './logger.js';
 
 export const defaultBrowser = 'chrome';
 
-export const defaultConfig: Omit<Config, 'gridUrl' | 'testsDir' | 'tsConfig' | 'webdriver'> = {
+export const defaultConfig: Omit<Config, 'gridUrl' | 'tsConfig' | 'webdriver'> = {
   disableTelemetry: false,
   useWorkerQueue: false,
   useDocker: true,
-  dockerImage: 'aerokube/selenoid:latest-release',
+  dockerImage: 'aerokube/selenoid:latest', // TODO What about playwright?
   dockerImagePlatform: '',
   pullImages: true,
   failFast: false,
   storybookUrl: 'http://localhost:6006',
   screenDir: path.resolve('images'),
   reportDir: path.resolve('report'),
+  testsDir: path.resolve('src'),
   reporter: process.env.TEAMCITY_VERSION ? TeamcityReporter : CreeveyReporter,
-  storiesProvider: browserStoriesProvider,
+  storiesProvider: hybridStoriesProvider,
   maxRetries: 0,
   testTimeout: 30000,
   diffOptions: { threshold: 0.1, includeAA: false },
   odiffOptions: { threshold: 0.1, antialiasing: true },
   browsers: { [defaultBrowser]: true },
   hooks: {},
-  testsRegex: /\.creevey\.(t|j)s$/,
+  testsRegex: /\.creevey\.(m|c)?(t|j)s$/,
 };
 
 function normalizeBrowserConfig(name: string, config: BrowserConfig): BrowserConfigObject {
@@ -57,7 +69,7 @@ function resolveConfigPath(configPath?: string): string | undefined {
   return configPath;
 }
 
-export async function readConfig(options: Options): Promise<Config> {
+export async function readConfig(options: Options | WorkerOptions): Promise<Config> {
   const configPath = resolveConfigPath(options.config);
   const userConfig: typeof defaultConfig & Partial<Pick<Config, 'gridUrl' | 'storiesProvider'>> = { ...defaultConfig };
 
@@ -80,14 +92,45 @@ export async function readConfig(options: Options): Promise<Config> {
       configData.webdriver = SeleniumWebdriver;
     }
 
+    for (const key in configData) {
+      const configKey = key as keyof typeof configData;
+      if (configData[configKey] === undefined) {
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete configData[configKey];
+      }
+    }
+
     Object.assign(userConfig, configData);
   }
 
-  if (options.failFast != undefined) userConfig.failFast = Boolean(options.failFast);
+  if (userConfig.resolveStorybookUrl && !options.storybookUrl) {
+    userConfig.storybookUrl = await userConfig.resolveStorybookUrl();
+  }
+
   if (options.reportDir) userConfig.reportDir = path.resolve(options.reportDir);
   if (options.screenDir) userConfig.screenDir = path.resolve(options.screenDir);
   if (options.storybookUrl) userConfig.storybookUrl = options.storybookUrl;
 
+  if (v.is(OptionsSchema, options)) {
+    if (options.docker === false) userConfig.useDocker = false;
+    if (options.failFast != undefined) userConfig.failFast = Boolean(options.failFast);
+    if (cluster.isPrimary) {
+      if (options.storybookPort) {
+        const url = new URL(userConfig.storybookUrl);
+        url.port = `${options.storybookPort}`;
+        userConfig.storybookUrl = url.toString();
+      }
+      if (typeof options.storybookStart === 'string') userConfig.storybookAutorunCmd = options.storybookStart;
+      else if (options.storybookStart) {
+        const { default: getPort } = await import('get-port');
+        const url = new URL(userConfig.storybookUrl);
+        const port = await getPort({ port: Number(url.port) });
+        url.port = `${port}`;
+        userConfig.storybookUrl = url.toString();
+      }
+    }
+  }
+
   // NOTE: Hack to pass typescript checking
   const config = userConfig as Config;
 

package/src/server/connection.ts

@@ -0,0 +1,26 @@
+import type { Config, Options } from '../types';
+import { waitOnUrl } from './utils.js';
+import { logger } from './logger.js';
+
+const RESPONSE_CHECK_TIMEOUT_MS = 10000;
+const RESPONSE_CHECK_INTERVAL_MS = 200;
+
+export function getStorybookUrl({ storybookUrl }: Config, { storybookStart }: Options): [string, string | undefined] {
+  if (storybookStart) {
+    const url = new URL(storybookUrl);
+    url.hostname = 'localhost';
+    return [url.toString(), storybookUrl];
+  }
+  return [storybookUrl, undefined];
+}
+
+export async function checkIsStorybookConnected(url: string) {
+  try {
+    await waitOnUrl(url, RESPONSE_CHECK_TIMEOUT_MS, RESPONSE_CHECK_INTERVAL_MS);
+    return true;
+  } catch (reason: unknown) {
+    const error = reason instanceof Error ? (reason.stack ?? reason.message) : (reason as string);
+    logger().error(error);
+    return false;
+  }
+}

package/src/server/docker.ts

@@ -1,9 +1,10 @@
 import tar from 'tar-stream';
+import Logger from 'loglevel';
 import { Writable } from 'stream';
 import Dockerode, { Container } from 'dockerode';
 import { DockerAuth } from '../types.js';
-import { subscribeOn } from './messages.js';
 import { logger } from './logger.js';
+import { setWorkerContainer } from './worker/context.js';
 
 const docker = new Dockerode();
 
@@ -50,17 +51,45 @@ export async function pullImages(
         function onProgress(event: { id: string; status: string; progress?: string }): void {
           if (!/^[a-z0-9]{12}$/i.test(event.id)) return;
 
-          spinner.text = `${image}: [${event.id}] ${event.status} ${event.progress ? event.progress : ''}`;
+          spinner.text = `${image}: [${event.id}] ${event.status} ${event.progress ?? ''}`;
         }
       });
     });
   }
 }
 
-export async function buildImage(imageName: string, dockerfile: string): Promise<void> {
+export async function buildImage(imageName: string, version: string, dockerfile: string): Promise<void> {
   const images = await docker.listImages({ filters: { label: [`creevey=${imageName}`] } });
 
-  if (images.at(0)) {
+  const containers = await docker.listContainers({ all: true, filters: { label: [`creevey=${imageName}`] } });
+  if (containers.length > 0) {
+    await Promise.all(
+      containers.map(async (info) => {
+        const container = docker.getContainer(info.Id);
+        try {
+          await container.remove({ force: true });
+        } catch {
+          /* noop */
+        }
+      }),
+    );
+  }
+
+  const oldImages = images.filter((info) => info.Labels.version !== version);
+  if (oldImages.length > 0) {
+    await Promise.all(
+      oldImages.map(async (info) => {
+        const image = docker.getImage(info.Id);
+        try {
+          await image.remove({ force: true });
+        } catch {
+          /* noop */
+        }
+      }),
+    );
+  }
+
+  if (oldImages.length !== images.length) {
     logger().info(`Image ${imageName} already exists`);
     return;
   }
@@ -70,15 +99,19 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
   pack.finalize();
 
   const { default: yoctoSpinner } = await import('yocto-spinner');
-  const spinner = yoctoSpinner({ text: `${imageName}: Build start` }).start();
+  const spinner = yoctoSpinner({ text: `${imageName}: Build start` });
+  if (logger().getLevel() > Logger.levels.DEBUG) {
+    spinner.start();
+  }
+  let isFailed = false;
   await new Promise<void>((resolve, reject) => {
-    void docker.buildImage(
-      // @ts-expect-error Type incompatibility AsyncIterator and AsyncIterableIterator
+    docker.buildImage(
       pack,
-      { t: imageName, labels: { creevey: imageName } },
+      // TODO Support buildkit decode grpc (version: '2')
+      { t: imageName, labels: { creevey: imageName, version }, version: '1' },
       (buildError: Error | null, stream) => {
         if (buildError || !stream) {
-          spinner.error(buildError?.message);
+          // spinner.error(buildError?.message);
           reject(buildError ?? new Error('Unknown error'));
           return;
         }
@@ -86,6 +119,8 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
         docker.modem.followProgress(stream, onFinished, onProgress);
 
         function onFinished(error: Error | null): void {
+          if (isFailed) return;
+
           if (error) {
             spinner.error(error.message);
             reject(error);
@@ -95,10 +130,23 @@ export async function buildImage(imageName: string, dockerfile: string): Promise
           resolve();
         }
 
-        function onProgress(event: { id: string; status: string; progress?: string }): void {
-          if (!/^[a-z0-9]{12}$/i.test(event.id)) return;
-
-          spinner.text = `${imageName}: [${event.id}] ${event.status} ${event.progress ? event.progress : ''}`;
+        function onProgress(
+          event:
+            | { stream: string }
+            | { errorDetail: { code: number; message: string }; error: string }
+            | { id: string; aux: string }, // NOTE: Only with `version: '2'`
+        ): void {
+          if ('stream' in event) {
+            if (logger().getLevel() <= Logger.levels.DEBUG) {
+              logger().debug(event.stream.trim());
+            } else {
+              spinner.text = `${imageName}: [Build] - ${event.stream}`;
+            }
+          } else if ('errorDetail' in event) {
+            isFailed = true;
+            spinner.error(event.error);
+            reject(new Error(event.error));
+          }
         }
       },
     );
@@ -111,33 +159,13 @@ export async function runImage(
   options: Record<string, unknown>,
   debug: boolean,
 ): Promise<string> {
-  await Promise.all(
-    (await docker.listContainers({ all: true, filters: { ancestor: [image] } })).map(async (info) => {
-      const container = docker.getContainer(info.Id);
-      try {
-        await container.stop();
-      } catch {
-        /* noop */
-      }
-      await container.remove();
-    }),
-  );
-
   const hub = docker.run(image, args, debug ? process.stdout : new DevNull(), options, (error) => {
     if (error) throw error;
   });
 
   return new Promise((resolve) => {
     hub.once('container', (container: Container) => {
-      // eslint-disable-next-line @typescript-eslint/no-misused-promises
-      subscribeOn('shutdown', async () => {
-        try {
-          await container.stop();
-          await container.remove();
-        } catch {
-          /* noop */
-        }
-      });
+      setWorkerContainer(container);
     });
     hub.once(
       'start',