creevey 0.9.1 → 0.10.0-beta.0

package/src/server/worker/match-image.ts

@@ -0,0 +1,303 @@
+import path from 'path';
+import { Stats } from 'fs';
+import { mkdir, readdir, readFile, stat, writeFile } from 'fs/promises';
+import { compare } from 'odiff-bin';
+import { PNG } from 'pngjs';
+import { Config, DiffOptions, Images, ImagesError } from '../../types.js';
+import assert from 'assert';
+
+export interface ImageContext {
+  attachments: string[];
+  testFullPath: string[];
+  images: Partial<Record<string, Images>>;
+}
+
+interface ImagePaths {
+  imageName: string;
+  actualImageName: string;
+  expectImageName: string;
+  diffImageName: string;
+  expectImageDir: string;
+  reportImageDir: string;
+}
+
+async function getStat(filePath: string): Promise<Stats | null> {
+  try {
+    return await stat(filePath);
+  } catch (error) {
+    if (typeof error == 'object' && error && (error as { code?: unknown }).code === 'ENOENT') {
+      return null;
+    }
+    throw error;
+  }
+}
+
+async function getLastImageNumber(imageDir: string, imageName: string): Promise<number> {
+  const actualImagesRegexp = new RegExp(`${imageName}-actual-(\\d+)\\.png`);
+
+  try {
+    return (
+      (await readdir(imageDir))
+        .map((filename) => filename.replace(actualImagesRegexp, '$1'))
+        .map(Number)
+        .filter((x) => !isNaN(x))
+        .sort((a, b) => b - a)[0] ?? 0
+    );
+  } catch (_error) {
+    return 0;
+  }
+}
+
+async function readExpected(expectImageDir: string, imageName: string): Promise<Buffer> {
+  const expected = await readFile(path.join(expectImageDir, `${imageName}.png`));
+
+  return expected;
+}
+
+async function saveImages(imageDir: string, images: { name: string; data: Buffer }[]): Promise<string[]> {
+  const files: string[] = [];
+  await mkdir(imageDir, { recursive: true });
+  for (const { name, data } of images) {
+    const filePath = path.join(imageDir, name);
+    await writeFile(filePath, data);
+    files.push(filePath);
+  }
+  return files;
+}
+
+async function getImagePaths(config: Config, testFullPath: string[], assertImageName?: string): Promise<ImagePaths> {
+  const testPath = [...testFullPath];
+  const imageName = assertImageName ?? testPath.pop();
+
+  assert(typeof imageName === 'string', `Can't get image name from empty test scope`);
+
+  const expectImageDir = path.join(config.screenDir, ...testPath);
+  const reportImageDir = path.join(config.reportDir, ...testPath);
+  const imageNumber = (await getLastImageNumber(reportImageDir, imageName)) + 1;
+  const actualImageName = `${imageName}-actual-${imageNumber}.png`;
+  const expectImageName = `${imageName}-expect-${imageNumber}.png`;
+  const diffImageName = `${imageName}-diff-${imageNumber}.png`;
+
+  return { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir };
+}
+
+async function getExpected(
+  ctx: ImageContext,
+  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
+): Promise<{
+  expected: Buffer | null;
+  onCompare: (actual: Buffer, expect?: Buffer, diff?: Buffer) => Promise<void>;
+}> {
+  const onCompare = async (actual: Buffer, expect?: Buffer, diff?: Buffer): Promise<void> => {
+    const imagesMeta: { name: string; data: Buffer }[] = [];
+    const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
+
+    imagesMeta.push({ name: image.actual, data: actual });
+
+    if (diff && expect) {
+      image.expect = expectImageName;
+      image.diff = diffImageName;
+      imagesMeta.push({ name: image.expect, data: expect });
+      imagesMeta.push({ name: image.diff, data: diff });
+    }
+    ctx.attachments = await saveImages(reportImageDir, imagesMeta);
+  };
+
+  const expectImageStat = await getStat(path.join(expectImageDir, `${imageName}.png`));
+  if (!expectImageStat) return { expected: null, onCompare };
+
+  const expected = await readExpected(expectImageDir, imageName);
+
+  return { expected, onCompare };
+}
+
+async function getOdiffExpected(
+  ctx: ImageContext,
+  actual: Buffer,
+  { imageName, actualImageName, expectImageName, diffImageName, expectImageDir, reportImageDir }: ImagePaths,
+): Promise<{ actual: string; expect: string; diff: string }> {
+  const expected = await readExpected(expectImageDir, imageName);
+
+  const image = (ctx.images[imageName] = ctx.images[imageName] ?? { actual: actualImageName });
+  image.expect = expectImageName;
+  image.diff = diffImageName;
+
+  const imagesMeta = [
+    { name: image.actual, data: actual },
+    { name: expectImageName, data: expected },
+  ];
+
+  ctx.attachments = await saveImages(reportImageDir, imagesMeta);
+
+  return {
+    actual: path.join(reportImageDir, actualImageName),
+    expect: path.join(reportImageDir, expectImageName),
+    diff: path.join(reportImageDir, diffImageName),
+  };
+}
+
+function normalizeImageSize(image: PNG, width: number, height: number): Buffer {
+  const normalizedImage = Buffer.alloc(4 * width * height);
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const i = (y * width + x) * 4;
+      if (x < image.width && y < image.height) {
+        const j = (y * image.width + x) * 4;
+        normalizedImage[i + 0] = image.data[j + 0];
+        normalizedImage[i + 1] = image.data[j + 1];
+        normalizedImage[i + 2] = image.data[j + 2];
+        normalizedImage[i + 3] = image.data[j + 3];
+      } else {
+        normalizedImage[i + 0] = 0;
+        normalizedImage[i + 1] = 0;
+        normalizedImage[i + 2] = 0;
+        normalizedImage[i + 3] = 0;
+      }
+    }
+  }
+  return normalizedImage;
+}
+
+function hasDiffPixels(diff: Buffer): boolean {
+  for (let i = 0; i < diff.length; i += 4) {
+    if (diff[i + 0] == 255 && diff[i + 1] == 0 && diff[i + 2] == 0 && diff[i + 3] == 255) return true;
+  }
+  return false;
+}
+
+function compareImages(
+  expect: Buffer,
+  actual: Buffer,
+  pixelmatch: typeof import('pixelmatch'),
+  diffOptions: DiffOptions,
+): { isEqual: boolean; diff: Buffer } {
+  const expectImage = PNG.sync.read(expect);
+  const actualImage = PNG.sync.read(actual);
+
+  const width = Math.max(actualImage.width, expectImage.width);
+  const height = Math.max(actualImage.height, expectImage.height);
+
+  const diffImage = new PNG({ width, height });
+
+  let actualImageData = actualImage.data;
+  if (actualImage.width < width || actualImage.height < height) {
+    actualImageData = normalizeImageSize(actualImage, width, height);
+  }
+
+  let expectImageData = expectImage.data;
+  if (expectImage.width < width || expectImage.height < height) {
+    expectImageData = normalizeImageSize(expectImage, width, height);
+  }
+
+  pixelmatch(expectImageData, actualImageData, diffImage.data, width, height, diffOptions);
+
+  return {
+    isEqual: !hasDiffPixels(diffImage.data),
+    diff: PNG.sync.write(diffImage),
+  };
+}
+
+export async function getMatchers(ctx: ImageContext, config: Config) {
+  // TODO Replace with `import from`
+  const { default: pixelmatch } = await import('pixelmatch');
+
+  async function assertImage(actual: Buffer, imageName?: string): Promise<string | undefined> {
+    const { expected, onCompare } = await getExpected(ctx, await getImagePaths(config, ctx.testFullPath, imageName));
+
+    if (expected == null) {
+      await onCompare(actual);
+      return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
+    }
+
+    if (actual.equals(expected)) {
+      await onCompare(actual);
+      return;
+    }
+
+    const { isEqual, diff } = compareImages(expected, actual, pixelmatch, config.diffOptions);
+
+    if (isEqual) {
+      await onCompare(actual);
+      return;
+    }
+
+    await onCompare(actual, expected, diff);
+
+    return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
+  }
+
+  return {
+    matchImage: async (image: Buffer, imageName?: string) => {
+      const errorMessage = await assertImage(image, imageName);
+      if (errorMessage) {
+        throw createImageError(imageName ? { [imageName]: errorMessage } : errorMessage);
+      }
+    },
+    matchImages: async (images: Record<string, Buffer>) => {
+      const errors: Record<string, string> = {};
+      await Promise.all(
+        Object.entries(images).map(async ([imageName, image]) => {
+          const errorMessage = await assertImage(image, imageName);
+          if (errorMessage) {
+            errors[imageName] = errorMessage;
+          }
+        }),
+      );
+      if (Object.keys(errors).length > 0) {
+        throw createImageError(errors);
+      }
+    },
+  };
+}
+
+function createImageError(imageErrors: string | Partial<Record<string, string>>): ImagesError {
+  const error = new ImagesError('Expected image to match');
+  error.images = imageErrors;
+  return error;
+}
+
+export function getOdiffMatchers(ctx: ImageContext, config: Config) {
+  const diffOptions = {
+    ...config.odiffOptions,
+    noFailOnFsErrors: true,
+  };
+
+  async function assertImage(image: Buffer, imageName?: string): Promise<string | undefined> {
+    const { actual, expect, diff } = await getOdiffExpected(
+      ctx,
+      image,
+      await getImagePaths(config, ctx.testFullPath, imageName),
+    );
+    const result = await compare(actual, expect, diff, diffOptions);
+    if (!result.match) {
+      if (result.reason == 'file-not-exists') {
+        return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
+      }
+      return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
+    }
+  }
+
+  return {
+    matchImage: async (image: Buffer, imageName?: string) => {
+      const errorMessage = await assertImage(image, imageName);
+      if (errorMessage) {
+        throw createImageError(imageName ? { [imageName]: errorMessage } : errorMessage);
+      }
+    },
+    matchImages: async (images: Record<string, Buffer>) => {
+      const errors: Record<string, string> = {};
+      await Promise.all(
+        Object.entries(images).map(async ([imageName, image]) => {
+          const errorMessage = await assertImage(image, imageName);
+          if (errorMessage) {
+            errors[imageName] = errorMessage;
+          }
+        }),
+      );
+      if (Object.keys(errors).length > 0) {
+        throw createImageError(errors);
+      }
+    },
+  };
+}

package/src/server/worker/start.ts

@@ -0,0 +1,303 @@
+import chai from 'chai';
+import chalk from 'chalk';
+import Logger from 'loglevel';
+import EventEmitter from 'events';
+import { Key, until, WebDriver } from 'selenium-webdriver';
+import {
+  BaseCreeveyTestContext,
+  Config,
+  CreeveyWebdriver,
+  FakeSuite,
+  FakeTest,
+  Images,
+  Options,
+  ServerTest,
+  TEST_EVENTS,
+  TestMessage,
+  isDefined,
+  isImageError,
+} from '../../types.js';
+import { subscribeOn, emitTestMessage, emitWorkerMessage } from '../messages.js';
+import chaiImage from './chai-image.js';
+import { SeleniumWebdriver } from '../selenium/webdriver.js';
+import { getMatchers, getOdiffMatchers, ImageContext } from './match-image.js';
+import { loadTestsFromStories } from '../stories.js';
+import { logger } from '../logger.js';
+import { getTestPath } from '../utils.js';
+
+async function getTestsFromStories(
+  config: Config,
+  browserName: string,
+  webdriver: CreeveyWebdriver,
+): Promise<Map<string, ServerTest>> {
+  const testsById = new Map<string, ServerTest>();
+  const tests = await loadTestsFromStories(
+    [browserName],
+    (listener) => config.storiesProvider(config, listener, webdriver),
+    (testsDiff) => {
+      Object.entries(testsDiff).forEach(([id, newTest]) => {
+        if (newTest) testsById.set(id, newTest);
+        else testsById.delete(id);
+      });
+    },
+  );
+
+  Object.values(tests)
+    .filter(isDefined)
+    .forEach((test) => testsById.set(test.id, test));
+
+  return testsById;
+}
+
+async function outputTraceLogs(browser: WebDriver, test: ServerTest, logger: Logger.Logger): Promise<void> {
+  const output: string[] = [];
+  const types = await browser.manage().logs().getAvailableLogTypes();
+  for (const type of types) {
+    const logs = await browser.manage().logs().get(type);
+    output.push(logs.map((log) => JSON.stringify(log.toJSON(), null, 2)).join('\n'));
+  }
+  logger.debug(
+    '----------',
+    getTestPath(test).join('/'),
+    '----------\n',
+    output.join('\n'),
+    '\n----------------------------------------------------------------------------------------------------',
+  );
+}
+
+function runHandler(browserName: string, images: Partial<Record<string, Images>>, error?: unknown): void {
+  // TODO How handle browser corruption?
+  if (isImageError(error)) {
+    if (typeof error.images == 'string') {
+      const image = images[browserName];
+      if (image) image.error = error.images;
+    } else {
+      const imageErrors = error.images ?? {};
+      Object.keys(imageErrors).forEach((imageName) => {
+        const image = images[imageName];
+        if (image) image.error = imageErrors[imageName];
+      });
+    }
+  }
+
+  if (error || Object.values(images).some((image) => image?.error != null)) {
+    const errorMessage = serializeError(error);
+
+    const isUnexpectedError =
+      hasTimeout(errorMessage) ||
+      hasDisconnected(errorMessage) ||
+      Object.values(images).some((image) => hasTimeout(image?.error));
+    if (isUnexpectedError) emitWorkerMessage({ type: 'error', payload: { subtype: 'unknown', error: errorMessage } });
+    else
+      emitTestMessage({
+        type: 'end',
+        payload: {
+          status: 'failed',
+          images,
+          error: errorMessage,
+        },
+      });
+  } else {
+    emitTestMessage({ type: 'end', payload: { status: 'success', images } });
+  }
+}
+
+async function setupWebdriver(webdriver: CreeveyWebdriver): Promise<[string, CreeveyWebdriver] | undefined> {
+  if ((await webdriver.openBrowser(true)) == null) {
+    logger.error('Failed to start browser');
+    emitWorkerMessage({
+      type: 'error',
+      payload: { subtype: 'browser', error: 'Failed to start browser' },
+    });
+    return;
+  }
+
+  const sessionId = await webdriver.getSessionId();
+
+  return [sessionId, webdriver];
+}
+
+function serializeError(error: unknown): string {
+  if (!error) return 'Unknown error';
+  if (error instanceof Error) return error.stack ?? error.message;
+  return typeof error === 'object' ? JSON.stringify(error) : (error as string);
+}
+
+function hasDisconnected(str: string | null | undefined): boolean {
+  return str?.toLowerCase().includes('disconnected') ?? false;
+}
+
+function hasTimeout(str: string | null | undefined): boolean {
+  return str?.toLowerCase().includes('timeout') ?? false;
+}
+
+export async function start(browser: string, gridUrl: string, config: Config, options: Options): Promise<void> {
+  let retries = 0;
+  const imagesContext: ImageContext = {
+    attachments: [],
+    testFullPath: [],
+    images: {},
+  };
+  const Webdriver = config.webdriver;
+  const [sessionId, webdriver] = (await setupWebdriver(new Webdriver(browser, gridUrl, config, options))) ?? [];
+
+  if (!webdriver || !sessionId) return;
+
+  const workerLogger = Logger.getLogger(`${browser}:${chalk.gray(sessionId)}`);
+
+  const reporterOptions = {
+    ...config.reporterOptions,
+    creevey: {
+      sessionId,
+      reportDir: config.reportDir,
+      browserName: browser,
+      get willRetry() {
+        return retries < config.maxRetries;
+      },
+      get images() {
+        return imagesContext.images;
+      },
+    },
+  };
+
+  class FakeRunner extends EventEmitter {}
+  const runner = new FakeRunner();
+  const Reporter = config.reporter;
+  new Reporter(runner, { reporterOptions });
+
+  const { matchImage, matchImages } = options.odiff
+    ? getOdiffMatchers(imagesContext, config)
+    : await getMatchers(imagesContext, config);
+  chai.use(chaiImage(matchImage, matchImages, workerLogger));
+
+  const tests = await (async () => {
+    try {
+      return await getTestsFromStories(config, browser, webdriver);
+    } catch (error) {
+      workerLogger.error('Failed to get tests from stories:', error);
+      emitWorkerMessage({
+        type: 'error',
+        payload: { subtype: 'browser', error: serializeError(error) },
+      });
+      return null;
+    }
+  })();
+
+  if (!tests) return;
+
+  subscribeOn('test', (message: TestMessage) => {
+    if (message.type != 'start') return;
+
+    const test = tests.get(message.payload.id);
+
+    if (!test) {
+      const error = `Test with id ${message.payload.id} not found`;
+      workerLogger.error(error);
+      emitWorkerMessage({
+        type: 'error',
+        payload: { subtype: 'test', error },
+      });
+      return;
+    }
+
+    const baseContext: BaseCreeveyTestContext = {
+      browserName: browser,
+      // TODO Pass browser instance to test
+      // browser: browser,
+      screenshots: [],
+
+      matchImage: matchImage,
+      matchImages: matchImages,
+
+      // NOTE: Deprecated
+      expect: chai.expect,
+      //TODO Move things below to the separate module
+      until: until,
+      keys: Key,
+    };
+
+    imagesContext.attachments = [];
+    imagesContext.testFullPath = getTestPath(test);
+    imagesContext.images = {};
+
+    retries = message.payload.retries;
+    let error = undefined;
+
+    const fakeSuite: FakeSuite = {
+      title: test.storyPath.slice(0, -1).join('/'),
+      fullTitle: () => fakeSuite.title,
+      titlePath: () => [fakeSuite.title],
+      tests: [],
+    };
+
+    const fakeTest: FakeTest = {
+      parent: fakeSuite,
+      title: [test.story.name, test.testName, test.browser].filter(isDefined).join('/'),
+      fullTitle: () => getTestPath(test).join('/'),
+      titlePath: () => getTestPath(test),
+      currentRetry: () => retries,
+      retires: () => config.maxRetries,
+      slow: () => 1000,
+    };
+
+    fakeSuite.tests.push(fakeTest);
+
+    void (async () => {
+      runner.emit(TEST_EVENTS.RUN_BEGIN);
+      runner.emit(TEST_EVENTS.TEST_BEGIN, fakeTest);
+
+      const start = Date.now();
+      try {
+        await Promise.race([
+          new Promise((reject) =>
+            setTimeout(() => {
+              reject(`Timeout of ${config.testTimeout}ms exceeded`);
+            }, config.testTimeout),
+          ),
+          (async () => {
+            const context = await webdriver.switchStory(test.story, baseContext, workerLogger);
+            await test.fn(context);
+          })(),
+        ]);
+      } catch (testError) {
+        error = testError;
+        fakeTest.err = error;
+      }
+      const duration = Date.now() - start;
+      fakeTest.attachments = imagesContext.attachments;
+      fakeTest.state = error ? 'failed' : 'passed';
+      fakeTest.duration = duration;
+      fakeTest.speed = duration > fakeTest.slow() ? 'slow' : duration / 2 > fakeTest.slow() ? 'medium' : 'fast';
+
+      if (error) {
+        runner.emit(TEST_EVENTS.TEST_FAIL, fakeTest, error);
+      } else {
+        runner.emit(TEST_EVENTS.TEST_PASS, fakeTest);
+      }
+      runner.emit(TEST_EVENTS.TEST_END, fakeTest);
+      runner.emit(TEST_EVENTS.RUN_END);
+
+      if (options.trace) {
+        try {
+          if (webdriver instanceof SeleniumWebdriver && webdriver.browser) {
+            await outputTraceLogs(webdriver.browser, test, workerLogger);
+          }
+        } catch (_) {
+          /* noop */
+        }
+      }
+
+      runHandler(baseContext.browserName, imagesContext.images, error);
+    })().catch((error: unknown) => {
+      workerLogger.error('Unexpected error:', error);
+      emitWorkerMessage({
+        type: 'error',
+        payload: { subtype: 'test', error: serializeError(error) },
+      });
+    });
+  });
+
+  workerLogger.info('Browser is ready');
+
+  emitWorkerMessage({ type: 'ready' });
+}