creevey 0.10.0-beta.8 → 0.10.0-rc.0

package/src/server/worker/match-image.ts

@@ -1,244 +1,28 @@
-import path from 'path';
-import { Stats } from 'fs';
-import { mkdir, readdir, readFile, stat, writeFile } from 'fs/promises';
-import { compare } from 'odiff-bin';
-import { PNG } from 'pngjs';
-import { Config, DiffOptions, Images, ImagesError } from '../../types.js';
-import assert from 'assert';
+import { Config, ImagesError } from '../../types.js';
+import { getOdiffAssert, getPixelmatchAssert, ImageContext } from '../compare.js';
 
-export interface ImageContext {
-  attachments: string[];
-  testFullPath: string[];
-  images: Partial<Record<string, Images>>;
-}
-
-interface ImagePaths {
-  imageName: string;
-  actualImageName: string;
-  expectImageName: string;
-  diffImageName: string;
-  expectImageDir: string;
-  reportImageDir: string;
-}
-
-async function getStat(filePath: string): Promise<Stats | null> {
-  try {
-    return await stat(filePath);
-  } catch (error) {
-    if (typeof error == 'object' && error && (error as { code?: unknown }).code === 'ENOENT') {
-      return null;
-    }
-    throw error;
-  }
-}
-
-async function getLastImageNumber(imageDir: string, imageName: string): Promise<number> {
-  const actualImagesRegexp = new RegExp(`${imageName}-actual-(\\d+)\\.png`);
-
-  try {
-    return (
-      (await readdir(imageDir))
-        .map((filename) => filename.replace(actualImagesRegexp, '$1'))
-        .map(Number)
-        .filter((x) => !isNaN(x))
-        .sort((a, b) => b - a)[0] ?? 0
-    );
-  } catch (_error) {
-    return 0;
-  }
-}
-
-async function readExpected(expectImageDir: string, imageName: string): Promise<Buffer> {
-  const expected = await readFile(path.join(expectImageDir, `${imageName}.png`));
-
-  return expected;
-}
-
-async function saveImages(imageDir: string, images: { name: string; data: Buffer }[]): Promise<string[]> {
-  const files: string[] = [];
-  await mkdir(imageDir, { recursive: true });
-  for (const { name, data } of images) {
-    const filePath = path.join(imageDir, name);
-    await writeFile(filePath, data);
-    files.push(filePath);
-  }
-  return files;
-}
-
-async function getImagePaths(config: Config, testFullPath: string[], assertImageName?: string): Promise<ImagePaths> {
-  const testPath = [...testFullPath];
-  const imageName = assertImageName ?? testPath.pop();
-
-  assert(typeof imageName === 'string', `Can't get image name from empty test scope`);
-
-  const expectImageDir = path.join(config.screenDir, ...testPath);
-  const reportImageDir = path.join(config.reportDir, ...testPath);
-  const imageNumber = (await getLastImageNumber(reportImageDir, imageName)) + 1;
-  const actualImageName = `${imageName}-actual-${imageNumber}.png`;
-  const expectImageName = `${imageName}-expect-${imageNumber}.png`;
-  const diffImageName = `${imageName}-diff-${imageNumber}.png`;
-
-  return { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir };
-}
-
-async function getExpected(
-  ctx: ImageContext,
-  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
-): Promise<{
-  expected: Buffer | null;
-  onCompare: (actual: Buffer, expect?: Buffer, diff?: Buffer) => Promise<void>;
-}> {
-  const onCompare = async (actual: Buffer, expect?: Buffer, diff?: Buffer): Promise<void> => {
-    const imagesMeta: { name: string; data: Buffer }[] = [];
-    const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
-
-    imagesMeta.push({ name: image.actual, data: actual });
-
-    if (diff && expect) {
-      image.expect = expectImageName;
-      image.diff = diffImageName;
-      imagesMeta.push({ name: image.expect, data: expect });
-      imagesMeta.push({ name: image.diff, data: diff });
-    }
-    ctx.attachments = await saveImages(reportImageDir, imagesMeta);
-  };
-
-  const expectImageStat = await getStat(path.join(expectImageDir, `${imageName}.png`));
-  if (!expectImageStat) return { expected: null, onCompare };
-
-  const expected = await readExpected(expectImageDir, imageName);
-
-  return { expected, onCompare };
-}
-
-async function getOdiffExpected(
-  ctx: ImageContext,
-  actual: Buffer,
-  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
-): Promise<{ actual: string; expect: string; diff: string }> {
-  const expected = await readExpected(expectImageDir, imageName);
-
-  const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
-  image.expect = expectImageName;
-  image.diff = diffImageName;
-
-  const imagesMeta = [
-    { name: image.actual, data: actual },
-    { name: expectImageName, data: expected },
-  ];
-
-  ctx.attachments = await saveImages(reportImageDir, imagesMeta);
-
-  return {
-    actual: path.join(reportImageDir, actualImageName),
-    expect: path.join(reportImageDir, expectImageName),
-    diff: path.join(reportImageDir, diffImageName),
-  };
-}
-
-function normalizeImageSize(image: PNG, width: number, height: number): Buffer {
-  const normalizedImage = Buffer.alloc(4 * width * height);
-
-  for (let y = 0; y < height; y++) {
-    for (let x = 0; x < width; x++) {
-      const i = (y * width + x) * 4;
-      if (x < image.width && y < image.height) {
-        const j = (y * image.width + x) * 4;
-        normalizedImage[i + 0] = image.data[j + 0];
-        normalizedImage[i + 1] = image.data[j + 1];
-        normalizedImage[i + 2] = image.data[j + 2];
-        normalizedImage[i + 3] = image.data[j + 3];
-      } else {
-        normalizedImage[i + 0] = 0;
-        normalizedImage[i + 1] = 0;
-        normalizedImage[i + 2] = 0;
-        normalizedImage[i + 3] = 0;
-      }
-    }
-  }
-  return normalizedImage;
-}
-
-function hasDiffPixels(diff: Buffer): boolean {
-  for (let i = 0; i < diff.length; i += 4) {
-    if (diff[i + 0] == 255 && diff[i + 1] == 0 && diff[i + 2] == 0 && diff[i + 3] == 255) return true;
-  }
-  return false;
-}
-
-function compareImages(
-  expect: Buffer,
-  actual: Buffer,
-  pixelmatch: typeof import('pixelmatch'),
-  diffOptions: DiffOptions,
-): { isEqual: boolean; diff: Buffer } {
-  const expectImage = PNG.sync.read(expect);
-  const actualImage = PNG.sync.read(actual);
-
-  const width = Math.max(actualImage.width, expectImage.width);
-  const height = Math.max(actualImage.height, expectImage.height);
-
-  const diffImage = new PNG({ width, height });
-
-  let actualImageData = actualImage.data;
-  if (actualImage.width < width || actualImage.height < height) {
-    actualImageData = normalizeImageSize(actualImage, width, height);
-  }
-
-  let expectImageData = expectImage.data;
-  if (expectImage.width < width || expectImage.height < height) {
-    expectImageData = normalizeImageSize(expectImage, width, height);
-  }
-
-  pixelmatch(expectImageData, actualImageData, diffImage.data, width, height, diffOptions);
-
-  return {
-    isEqual: !hasDiffPixels(diffImage.data),
-    diff: PNG.sync.write(diffImage),
-  };
+function toBuffer(bufferOrBase64: Buffer | string) {
+  return typeof bufferOrBase64 === 'string' ? Buffer.from(bufferOrBase64, 'base64') : bufferOrBase64;
 }
 
 export async function getMatchers(ctx: ImageContext, config: Config) {
   // TODO Replace with `import from`
   const { default: pixelmatch } = await import('pixelmatch');
 
-  async function assertImage(actual: Buffer, imageName?: string): Promise<string | undefined> {
-    const { expected, onCompare } = await getExpected(ctx, await getImagePaths(config, ctx.testFullPath, imageName));
-
-    if (expected == null) {
-      await onCompare(actual);
-      return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
-    }
-
-    if (actual.equals(expected)) {
-      await onCompare(actual);
-      return;
-    }
-
-    const { isEqual, diff } = compareImages(expected, actual, pixelmatch, config.diffOptions);
-
-    if (isEqual) {
-      await onCompare(actual);
-      return;
-    }
-
-    await onCompare(actual, expected, diff);
-
-    return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
-  }
+  const assertImage = getPixelmatchAssert(pixelmatch, ctx, config);
 
   return {
-    matchImage: async (image: Buffer, imageName?: string) => {
-      const errorMessage = await assertImage(image, imageName);
+    matchImage: async (image: Buffer | string, imageName?: string) => {
+      const errorMessage = await assertImage(toBuffer(image), imageName);
       if (errorMessage) {
         throw createImageError(imageName ? { [imageName]: errorMessage } : errorMessage);
       }
     },
-    matchImages: async (images: Record<string, Buffer>) => {
+    matchImages: async (images: Record<string, Buffer | string>) => {
       const errors: Record<string, string> = {};
       await Promise.all(
         Object.entries(images).map(async ([imageName, image]) => {
-          const errorMessage = await assertImage(image, imageName);
+          const errorMessage = await assertImage(toBuffer(image), imageName);
           if (errorMessage) {
             errors[imageName] = errorMessage;
           }
@@ -257,39 +41,23 @@ function createImageError(imageErrors: string | Partial<Record<string, string>>)
   return error;
 }
 
-export function getOdiffMatchers(ctx: ImageContext, config: Config) {
-  const diffOptions = {
-    ...config.odiffOptions,
-    noFailOnFsErrors: true,
-  };
+export async function getOdiffMatchers(ctx: ImageContext, config: Config) {
+  const { compare } = await import('odiff-bin');
 
-  async function assertImage(image: Buffer, imageName?: string): Promise<string | undefined> {
-    const { actual, expect, diff } = await getOdiffExpected(
-      ctx,
-      image,
-      await getImagePaths(config, ctx.testFullPath, imageName),
-    );
-    const result = await compare(actual, expect, diff, diffOptions);
-    if (!result.match) {
-      if (result.reason == 'file-not-exists') {
-        return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
-      }
-      return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
-    }
-  }
+  const assertImage = getOdiffAssert(compare, ctx, config);
 
   return {
-    matchImage: async (image: Buffer, imageName?: string) => {
-      const errorMessage = await assertImage(image, imageName);
+    matchImage: async (image: Buffer | string, imageName?: string) => {
+      const errorMessage = await assertImage(toBuffer(image), imageName);
       if (errorMessage) {
         throw createImageError(imageName ? { [imageName]: errorMessage } : errorMessage);
       }
     },
-    matchImages: async (images: Record<string, Buffer>) => {
+    matchImages: async (images: Record<string, Buffer | string>) => {
       const errors: Record<string, string> = {};
       await Promise.all(
         Object.entries(images).map(async ([imageName, image]) => {
-          const errorMessage = await assertImage(image, imageName);
+          const errorMessage = await assertImage(toBuffer(image), imageName);
           if (errorMessage) {
             errors[imageName] = errorMessage;
           }

package/src/server/worker/start.ts

@@ -1,25 +1,22 @@
 import chai from 'chai';
-import EventEmitter from 'events';
 import {
   BaseCreeveyTestContext,
   Config,
   CreeveyWebdriver,
-  FakeSuite,
-  FakeTest,
-  Images,
-  Options,
   ServerTest,
-  TEST_EVENTS,
   TestMessage,
+  TestResult,
+  WorkerOptions,
   isDefined,
   isImageError,
 } from '../../types.js';
 import { subscribeOn, emitTestMessage, emitWorkerMessage } from '../messages.js';
 import chaiImage from './chai-image.js';
-import { getMatchers, getOdiffMatchers, ImageContext } from './match-image.js';
+import { getMatchers, getOdiffMatchers } from './match-image.js';
 import { loadTestsFromStories } from '../stories.js';
 import { logger } from '../logger.js';
 import { getTestPath } from '../utils.js';
+import { ImageContext } from '../compare.js';
 
 async function getTestsFromStories(
   config: Config,
@@ -45,9 +42,10 @@ async function getTestsFromStories(
   return testsById;
 }
 
-function runHandler(browserName: string, images: Partial<Record<string, Images>>, error?: unknown): void {
+function runHandler(browserName: string, result: Omit<TestResult, 'status'>, error?: unknown): void {
   // TODO How handle browser corruption?
-  if (isImageError(error)) {
+  const { images } = result;
+  if (images != null && isImageError(error)) {
     if (typeof error.images == 'string') {
       const image = images[browserName];
       if (image) image.error = error.images;
@@ -60,25 +58,31 @@ function runHandler(browserName: string, images: Partial<Record<string, Images>>
     }
   }
 
-  if (error || Object.values(images).some((image) => image?.error != null)) {
-    const errorMessage = serializeError(error);
+  if (error || (images != null && Object.values(images).some((image) => image?.error != null))) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const errorMessage = result.error!;
 
     const isUnexpectedError =
       hasTimeout(errorMessage) ||
       hasDisconnected(errorMessage) ||
-      Object.values(images).some((image) => hasTimeout(image?.error));
+      (images != null && Object.values(images).some((image) => hasTimeout(image?.error)));
     if (isUnexpectedError) emitWorkerMessage({ type: 'error', payload: { subtype: 'unknown', error: errorMessage } });
     else
       emitTestMessage({
         type: 'end',
         payload: {
           status: 'failed',
-          images,
-          error: errorMessage,
+          ...result,
         },
       });
   } else {
-    emitTestMessage({ type: 'end', payload: { status: 'success', images } });
+    emitTestMessage({
+      type: 'end',
+      payload: {
+        status: 'success',
+        ...result,
+      },
+    });
   }
 }
 
@@ -111,8 +115,7 @@ function hasTimeout(str: string | null | undefined): boolean {
   return str?.toLowerCase().includes('timeout') ?? false;
 }
 
-export async function start(browser: string, gridUrl: string, config: Config, options: Options): Promise<void> {
-  let retries = 0;
+export async function start(browser: string, gridUrl: string, config: Config, options: WorkerOptions): Promise<void> {
   const imagesContext: ImageContext = {
     attachments: [],
     testFullPath: [],
@@ -123,28 +126,8 @@ export async function start(browser: string, gridUrl: string, config: Config, op
 
   if (!webdriver || !sessionId) return;
 
-  const reporterOptions = {
-    ...config.reporterOptions,
-    creevey: {
-      sessionId,
-      reportDir: config.reportDir,
-      browserName: browser,
-      get willRetry() {
-        return retries < config.maxRetries;
-      },
-      get images() {
-        return imagesContext.images;
-      },
-    },
-  };
-
-  class FakeRunner extends EventEmitter {}
-  const runner = new FakeRunner();
-  const Reporter = config.reporter;
-  new Reporter(runner, { reporterOptions });
-
   const { matchImage, matchImages } = options.odiff
-    ? getOdiffMatchers(imagesContext, config)
+    ? await getOdiffMatchers(imagesContext, config)
     : await getMatchers(imagesContext, config);
   chai.use(chaiImage(matchImage, matchImages));
 
@@ -196,39 +179,20 @@ export async function start(browser: string, gridUrl: string, config: Config, op
     imagesContext.testFullPath = getTestPath(test);
     imagesContext.images = {};
 
-    retries = message.payload.retries;
     let error = undefined;
 
-    const fakeSuite: FakeSuite = {
-      title: test.storyPath.slice(0, -1).join('/'),
-      fullTitle: () => fakeSuite.title,
-      titlePath: () => [fakeSuite.title],
-      tests: [],
-    };
-
-    const fakeTest: FakeTest = {
-      parent: fakeSuite,
-      title: [test.story.name, test.testName, test.browser].filter(isDefined).join('/'),
-      fullTitle: () => getTestPath(test).join('/'),
-      titlePath: () => getTestPath(test),
-      currentRetry: () => retries,
-      retires: () => config.maxRetries,
-      slow: () => 1000,
-    };
-
-    fakeSuite.tests.push(fakeTest);
-
     void (async () => {
-      runner.emit(TEST_EVENTS.RUN_BEGIN);
-      runner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
-
+      let timeout;
+      let isRejected = false;
       const start = Date.now();
       try {
         await Promise.race([
-          new Promise((reject) =>
-            setTimeout(() => {
-              reject(`Timeout of ${config.testTimeout}ms exceeded`);
-            }, config.testTimeout),
+          new Promise(
+            (_, reject) =>
+              (timeout = setTimeout(() => {
+                isRejected = true;
+                reject(new Error(`Timeout of ${config.testTimeout}ms exceeded`));
+              }, config.testTimeout)),
           ),
           (async () => {
             const context = await webdriver.switchStory(test.story, baseContext);
@@ -237,25 +201,31 @@ export async function start(browser: string, gridUrl: string, config: Config, op
         ]);
       } catch (testError) {
         error = testError;
-        fakeTest.err = error;
       }
       const duration = Date.now() - start;
-      fakeTest.attachments = imagesContext.attachments;
-      fakeTest.state = error ? 'failed' : 'passed';
-      fakeTest.duration = duration;
-      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
-
-      if (error) {
-        runner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, error);
-      } else {
-        runner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
-      }
-      runner.emit(TEST_EVENTS.TEST_END, fakeTest);
-      runner.emit(TEST_EVENTS.RUN_END);
+      clearTimeout(timeout);
 
       await webdriver.afterTest(test);
 
-      runHandler(baseContext.browserName, imagesContext.images, error);
+      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+      if (isRejected) {
+        emitWorkerMessage({
+          type: 'error',
+          payload: { subtype: 'unknown', error: serializeError(error) },
+        });
+      } else {
+        const result = {
+          sessionId,
+          browserName: baseContext.browserName,
+          workerId: process.pid,
+          images: imagesContext.images,
+          error: error ? serializeError(error) : undefined,
+          duration,
+          attachments: imagesContext.attachments,
+          retries: message.payload.retries,
+        };
+        runHandler(baseContext.browserName, result, error);
+      }
     })().catch((error: unknown) => {
       logger().error('Unexpected error:', error);
       emitWorkerMessage({

package/src/shared/index.ts

@@ -1,12 +1,14 @@
-import _ from 'lodash';
-import { Parameters } from '@storybook/csf';
+import mapValues from 'lodash/mapValues.js';
+import mergeWith from 'lodash/mergeWith.js';
+import cloneDeepWith from 'lodash/cloneDeepWith.js';
+import type { Parameters } from 'storybook/internal/types';
 import { SetStoriesData, StoriesRaw, CreeveyStoryParams, StoryInput } from '../types.js';
 import { deserializeRegExp, isSerializedRegExp, isRegExp, serializeRegExp } from './serializeRegExp.js';
 
 // NOTE: Copy-paste from storybook/api
 export const combineParameters = (...parameterSets: Parameters[]): Parameters =>
   // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-  _.mergeWith({}, ...parameterSets, (_: unknown, srcValue: unknown) => {
+  mergeWith({}, ...parameterSets, (_: unknown, srcValue: unknown) => {
     // Treat arrays as scalars:
     if (Array.isArray(srcValue)) return srcValue as unknown[];
 
@@ -19,14 +21,14 @@ export const denormalizeStoryParameters = ({
   kindParameters,
   stories,
 }: SetStoriesData): StoriesRaw => {
-  return _.mapValues(stories, (storyData) => ({
+  return mapValues(stories, (storyData) => ({
     ...storyData,
     parameters: combineParameters(globalParameters, kindParameters[storyData.title] ?? {}, storyData.parameters),
   })) as StoriesRaw;
 };
 
 export const serializeRawStories = (stories: StoriesRaw): StoriesRaw => {
-  return _.mapValues(stories, (storyData) => {
+  return mapValues(stories, (storyData) => {
     const creevey = storyData.parameters.creevey as CreeveyStoryParams | undefined;
     if (creevey?.skip) {
       return {
@@ -35,7 +37,7 @@ export const serializeRawStories = (stories: StoriesRaw): StoriesRaw => {
           ...storyData.parameters,
           creevey: {
             ...creevey,
-            skip: _.cloneDeepWith(creevey.skip, (value) => {
+            skip: cloneDeepWith(creevey.skip, (value) => {
               if (isRegExp(value)) {
                 return serializeRegExp(value);
               }
@@ -50,7 +52,7 @@ export const serializeRawStories = (stories: StoriesRaw): StoriesRaw => {
 };
 
 export const deserializeRawStories = (stories: StoriesRaw): StoriesRaw => {
-  return _.mapValues(stories, deserializeStory);
+  return mapValues(stories, deserializeStory);
 };
 
 export const deserializeStory = (story: StoryInput): StoryInput => {
@@ -62,7 +64,7 @@ export const deserializeStory = (story: StoryInput): StoryInput => {
         ...story.parameters,
         creevey: {
           ...creevey,
-          skip: _.cloneDeepWith(creevey.skip, (value) => {
+          skip: cloneDeepWith(creevey.skip, (value) => {
             if (isSerializedRegExp(value)) {
               return deserializeRegExp(value);
             }