creevey 0.9.0-beta.2 → 0.9.0-beta.21

package/src/server/update.ts

@@ -0,0 +1,74 @@
+import path from 'path';
+import { Dirent, mkdirSync, copyFileSync, readdirSync } from 'fs';
+import micromatch from 'micromatch';
+import { Config, isDefined } from '../types.js';
+import { tryToLoadTestsData } from './utils.js';
+
+const actualRegex = /^(.*)-actual-(\d+)\.png$/i;
+
+function approve(
+  dirents: Dirent[],
+  srcPath: string,
+  dstPath: string,
+  testPaths: string[][] | null,
+  isMatch: (value: string) => boolean,
+): void {
+  dirents
+    .filter((dirent) => dirent.isFile())
+    .map((dirent) => actualRegex.exec(dirent.name))
+    .filter(isDefined)
+    .filter(
+      ([fileName, imageName]) =>
+        !testPaths || (testPaths.find(([token]) => token == imageName) && isMatch(path.join(srcPath, fileName))),
+    )
+    .reduce(
+      (images, [, imageName, retry]) =>
+        Number(retry) > (images.get(imageName) ?? -1) ? images.set(imageName, Number(retry)) : images,
+      new Map<string, number>(),
+    )
+    .forEach((retry, imageName) => {
+      mkdirSync(dstPath, { recursive: true });
+      copyFileSync(path.join(srcPath, `${imageName}-actual-${retry}.png`), path.join(dstPath, `${imageName}.png`));
+    });
+}
+
+function traverse(
+  srcPath: string,
+  dstPath: string,
+  testPaths: string[][] | null,
+  isMatch: (value: string) => boolean,
+): void {
+  const dirents = readdirSync(srcPath, { withFileTypes: true });
+  approve(dirents, srcPath, dstPath, testPaths, isMatch);
+  dirents
+    .filter((dirent) => dirent.isDirectory())
+    .map(
+      (dirent) =>
+        [
+          dirent.name,
+          testPaths?.map(([token, ...restPath]) => (token == dirent.name ? restPath : null)).filter(isDefined),
+        ] as [string, string[][] | null],
+    )
+    .filter(([, paths]) => !paths || paths.length > 0)
+    .forEach(([dirname, paths]) => {
+      traverse(path.join(srcPath, dirname), path.join(dstPath, dirname), paths, isMatch);
+    });
+}
+
+export function update(config: Config, grepPattern?: string): void {
+  const { reportDir, screenDir } = config;
+  const isMatch = grepPattern ? micromatch.matcher(grepPattern, { contains: true }) : () => true;
+
+  const testsMeta = tryToLoadTestsData(`${reportDir}/tests.json`);
+  const testsReport = tryToLoadTestsData(`${reportDir}/data.js`);
+  let testPaths: string[][] | null = null;
+
+  if (testsMeta && testsReport) {
+    testPaths = Object.values(testsMeta)
+      .filter(isDefined)
+      .filter(({ id }) => testsReport[id]?.status == 'failed')
+      .map(({ storyPath, testName, browser }) => [...storyPath, ...(testName ? [testName] : []), browser]);
+  }
+
+  traverse(reportDir, screenDir, testPaths, (value: string) => isMatch(path.relative(reportDir, value)));
+}

package/src/server/utils.ts

@@ -0,0 +1,184 @@
+import fs from 'fs';
+import { get } from 'https';
+import cluster from 'cluster';
+import { dirname } from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { createRequire } from 'module';
+import findCacheDir from 'find-cache-dir';
+import { SkipOptions, SkipOption, isDefined, TestData, noop, ServerTest } from '../types.js';
+import { emitShutdownMessage, sendShutdownMessage } from './messages.js';
+
+const importMetaUrl = pathToFileURL(__filename).href;
+
+export const isShuttingDown = { current: false };
+
+export const LOCALHOST_REGEXP = /(localhost|127\.0\.0\.1)/i;
+
+export const configExt = ['.js', '.mjs', '.ts', '.cjs', '.mts', '.cts'];
+
+export const skipOptionKeys = ['in', 'kinds', 'stories', 'tests', 'reason'];
+
+function matchBy(pattern: string | string[] | RegExp | undefined, value: string): boolean {
+  return (
+    (typeof pattern == 'string' && pattern == value) ||
+    (Array.isArray(pattern) && pattern.includes(value)) ||
+    (pattern instanceof RegExp && pattern.test(value)) ||
+    !isDefined(pattern)
+  );
+}
+
+export function shouldSkip(
+  browser: string,
+  meta: {
+    title: string;
+    name: string;
+  },
+  skipOptions: SkipOptions,
+  test?: string,
+): string | boolean {
+  if (typeof skipOptions != 'object') {
+    return skipOptions;
+  }
+  for (const skipKey in skipOptions) {
+    const reason = shouldSkipByOption(browser, meta, skipOptions[skipKey], skipKey, test);
+    if (reason) return reason;
+  }
+  return false;
+}
+
+export function shouldSkipByOption(
+  browser: string,
+  meta: {
+    title: string;
+    name: string;
+  },
+  skipOption: SkipOption | SkipOption[],
+  reason: string,
+  test?: string,
+): string | boolean {
+  if (Array.isArray(skipOption)) {
+    for (const skip of skipOption) {
+      const result = shouldSkipByOption(browser, meta, skip, reason, test);
+      if (result) return result;
+    }
+    return false;
+  }
+
+  const { in: browsers, kinds, stories, tests } = skipOption;
+  const { title, name } = meta;
+  const skipByBrowser = matchBy(browsers, browser);
+  const skipByKind = matchBy(kinds, title);
+  const skipByStory = matchBy(stories, name);
+  const skipByTest = !isDefined(test) || matchBy(tests, test);
+
+  return skipByBrowser && skipByKind && skipByStory && skipByTest && reason;
+}
+
+export async function shutdownWorkers(): Promise<void> {
+  isShuttingDown.current = true;
+  await Promise.all(
+    Object.values(cluster.workers ?? {})
+      .filter(isDefined)
+      .filter((worker) => worker.isConnected())
+      .map(
+        (worker) =>
+          new Promise<void>((resolve) => {
+            const timeout = setTimeout(() => {
+              worker.kill();
+            }, 10000);
+            worker.on('exit', () => {
+              clearTimeout(timeout);
+              resolve();
+            });
+            sendShutdownMessage(worker);
+          }),
+      ),
+  );
+  emitShutdownMessage();
+}
+
+export function shutdown(): void {
+  process.exit();
+}
+
+export function getCreeveyCache(): string | undefined {
+  return findCacheDir({ name: 'creevey', cwd: dirname(fileURLToPath(importMetaUrl)) });
+}
+
+export async function runSequence(seq: (() => unknown)[], predicate: () => boolean): Promise<void> {
+  for (const fn of seq) {
+    if (predicate()) await fn();
+  }
+}
+
+export function testsToImages(tests: (TestData | undefined)[]): Set<string> {
+  return new Set(
+    ([] as string[]).concat(
+      ...tests
+        .filter(isDefined)
+        .map(({ browser, testName, storyPath, results }) =>
+          Object.keys(results?.slice(-1)[0]?.images ?? {}).map(
+            (image) =>
+              `${[...storyPath, testName, browser, browser == image ? undefined : image]
+                .filter(isDefined)
+                .join('/')}.png`,
+          ),
+        ),
+    ),
+  );
+}
+
+// https://tuhrig.de/how-to-know-you-are-inside-a-docker-container/
+export const isInsideDocker =
+  fs.existsSync('/proc/1/cgroup') && fs.readFileSync('/proc/1/cgroup', 'utf-8').includes('docker');
+
+export const downloadBinary = (downloadUrl: string, destination: string): Promise<void> =>
+  new Promise((resolve, reject) =>
+    get(downloadUrl, (response) => {
+      if (response.statusCode == 302) {
+        const { location } = response.headers;
+        if (!location) {
+          reject(new Error(`Couldn't download selenoid. Status code: ${response.statusCode ?? 'UNKNOWN'}`));
+          return;
+        }
+
+        resolve(downloadBinary(location, destination));
+        return;
+      }
+      if (response.statusCode != 200) {
+        reject(new Error(`Couldn't download selenoid. Status code: ${response.statusCode ?? 'UNKNOWN'}`));
+        return;
+      }
+
+      const fileStream = fs.createWriteStream(destination);
+      response.pipe(fileStream);
+
+      fileStream.on('finish', () => {
+        fileStream.close();
+        resolve();
+      });
+      fileStream.on('error', (error) => {
+        fs.unlink(destination, noop);
+        reject(error);
+      });
+    }),
+  );
+
+export function readDirRecursive(dirPath: string): string[] {
+  return ([] as string[]).concat(
+    ...fs
+      .readdirSync(dirPath, { withFileTypes: true })
+      .map((dirent) =>
+        dirent.isDirectory() ? readDirRecursive(`${dirPath}/${dirent.name}`) : [`${dirPath}/${dirent.name}`],
+      ),
+  );
+}
+
+const _require = createRequire(importMetaUrl);
+export function tryToLoadTestsData(filename: string): Partial<Record<string, ServerTest>> | undefined {
+  try {
+    return _require(filename) as Partial<Record<string, ServerTest>>;
+  } catch {
+    /* noop */
+  }
+}

package/src/server/worker/chai-image.ts

@@ -0,0 +1,147 @@
+import { PNG } from 'pngjs';
+import pixelmatch from 'pixelmatch';
+
+import { DiffOptions, ImagesError } from '../../types.js';
+
+function normalizeImageSize(image: PNG, width: number, height: number): Buffer {
+  const normalizedImage = Buffer.alloc(4 * width * height);
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const i = (y * width + x) * 4;
+      if (x < image.width && y < image.height) {
+        const j = (y * image.width + x) * 4;
+        normalizedImage[i + 0] = image.data[j + 0];
+        normalizedImage[i + 1] = image.data[j + 1];
+        normalizedImage[i + 2] = image.data[j + 2];
+        normalizedImage[i + 3] = image.data[j + 3];
+      } else {
+        normalizedImage[i + 0] = 0;
+        normalizedImage[i + 1] = 0;
+        normalizedImage[i + 2] = 0;
+        normalizedImage[i + 3] = 0;
+      }
+    }
+  }
+  return normalizedImage;
+}
+
+function hasDiffPixels(diff: Buffer): boolean {
+  for (let i = 0; i < diff.length; i += 4) {
+    if (diff[i + 0] == 255 && diff[i + 1] == 0 && diff[i + 2] == 0 && diff[i + 3] == 255) return true;
+  }
+  return false;
+}
+
+function compareImages(expect: Buffer, actual: Buffer, diffOptions: DiffOptions): { isEqual: boolean; diff: Buffer } {
+  const expectImage = PNG.sync.read(expect);
+  const actualImage = PNG.sync.read(actual);
+
+  const width = Math.max(actualImage.width, expectImage.width);
+  const height = Math.max(actualImage.height, expectImage.height);
+
+  const diffImage = new PNG({ width, height });
+
+  let actualImageData = actualImage.data;
+  if (actualImage.width < width || actualImage.height < height) {
+    actualImageData = normalizeImageSize(actualImage, width, height);
+  }
+
+  let expectImageData = expectImage.data;
+  if (expectImage.width < width || expectImage.height < height) {
+    expectImageData = normalizeImageSize(expectImage, width, height);
+  }
+
+  pixelmatch(expectImageData, actualImageData, diffImage.data, width, height, diffOptions);
+
+  return {
+    isEqual: !hasDiffPixels(diffImage.data),
+    diff: PNG.sync.write(diffImage),
+  };
+}
+
+export default function (
+  getExpected: (
+    imageName?: string,
+  ) => Promise<
+    | { expected: Buffer | null; onCompare: (actual: Buffer, expect?: Buffer, diff?: Buffer) => Promise<void> }
+    | Buffer
+    | null
+  >,
+  diffOptions: DiffOptions,
+) {
+  return function chaiImage({ Assertion }: Chai.ChaiStatic, utils: Chai.ChaiUtils): void {
+    async function assertImage(actual: Buffer, imageName?: string): Promise<string | undefined> {
+      let onCompare: (actual: Buffer, expect?: Buffer, diff?: Buffer) => Promise<void> = () => Promise.resolve();
+      let expected = await getExpected(imageName);
+      if (!(expected instanceof Buffer) && expected != null) ({ expected, onCompare } = expected);
+
+      if (expected == null) {
+        await onCompare(actual);
+        return imageName ? `Expected image '${imageName}' does not exists` : 'Expected image does not exists';
+      }
+
+      if (actual.equals(expected)) {
+        await onCompare(actual);
+        return;
+      }
+
+      const { isEqual, diff } = compareImages(expected, actual, diffOptions);
+
+      if (isEqual) {
+        await onCompare(actual);
+        return;
+      }
+
+      await onCompare(actual, expected, diff);
+
+      return imageName ? `Expected image '${imageName}' to match` : 'Expected image to match';
+    }
+
+    utils.addMethod(
+      Assertion.prototype,
+      'matchImage',
+      async function matchImage(this: Record<string, unknown>, imageName?: string) {
+        const actual = utils.flag(this, 'object') as string | Buffer;
+        const errorMessage = await assertImage(
+          typeof actual == 'string' ? Buffer.from(actual, 'base64') : actual,
+          imageName,
+        );
+        if (errorMessage) {
+          throw createImageError(imageName ? { [imageName]: errorMessage } : errorMessage);
+        }
+      },
+    );
+
+    utils.addMethod(Assertion.prototype, 'matchImages', async function matchImages(this: Record<string, unknown>) {
+      const errors: Record<string, string> = {};
+      await Promise.all(
+        Object.entries<string | Buffer>(utils.flag(this, 'object') as Record<string, string | Buffer>).map(
+          async ([imageName, imageOrBase64]) => {
+            let errorMessage: string | undefined;
+            try {
+              errorMessage = await assertImage(
+                typeof imageOrBase64 == 'string' ? Buffer.from(imageOrBase64, 'base64') : imageOrBase64,
+                imageName,
+              );
+            } catch (error) {
+              errorMessage = (error as Error).stack;
+            }
+            if (errorMessage) {
+              errors[imageName] = errorMessage;
+            }
+          },
+        ),
+      );
+      if (Object.keys(errors).length > 0) {
+        throw createImageError(errors);
+      }
+    });
+  };
+}
+
+function createImageError(imageErrors: string | Partial<Record<string, string>>): ImagesError {
+  const error = new Error('Expected image to match') as ImagesError;
+  error.images = imageErrors;
+  return error;
+}

package/src/server/worker/helpers.ts

@@ -0,0 +1,61 @@
+import { Suite, Context, Test } from 'mocha';
+import { Config, isDefined, ServerTest } from '../../types.js';
+import { loadTestsFromStories } from '../stories.js';
+
+function findOrCreateSuite(name: string, parent: Suite): Suite {
+  const suite = parent.suites.find(({ title }) => title == name) ?? new Suite(name, parent.ctx);
+  if (!suite.parent) {
+    suite.parent = parent;
+    parent.addSuite(suite);
+  }
+  return suite;
+}
+
+function createTest(name: string, fn: (this: Context) => Promise<void>, skip: string | boolean = false): Test {
+  const test = new Test(name, skip ? undefined : fn);
+  test.pending = Boolean(skip);
+  // NOTE Can't define skip reason in mocha https://github.com/mochajs/mocha/issues/2026
+  test.skipReason = skip;
+  return test;
+}
+
+function addTest(rootSuite: Suite, test: ServerTest): Test {
+  const [testName, ...suitePath] = [...test.storyPath, test.testName].reverse().filter(isDefined);
+  const suite = suitePath.reduceRight((subSuite, suiteName) => findOrCreateSuite(suiteName, subSuite), rootSuite);
+  const mochaTest = createTest(testName, test.fn, test.skip);
+  suite.addTest(mochaTest);
+  mochaTest.ctx = Object.setPrototypeOf({ id: test.id, story: test.story }, suite.ctx) as Context;
+  return mochaTest;
+}
+
+function removeTestOrSuite(testOrSuite: Test | Suite): void {
+  const { parent } = testOrSuite;
+  if (!parent) return;
+  if (testOrSuite instanceof Test) parent.tests = parent.tests.filter((test) => test != testOrSuite);
+  if (testOrSuite instanceof Suite) parent.suites = parent.suites.filter((suite) => suite != testOrSuite);
+  if (parent.tests.length == 0 && parent.suites.length == 0) removeTestOrSuite(parent);
+}
+
+export async function addTestsFromStories(
+  rootSuite: Suite,
+  config: Config,
+  { browser, ...options }: { browser: string; watch: boolean; debug: boolean; port: number },
+): Promise<void> {
+  const mochaTestsById = new Map<string, Test>();
+  const tests = await loadTestsFromStories(
+    [browser],
+    (listener) => config.storiesProvider(config, options, listener),
+    (testsDiff) => {
+      Object.entries(testsDiff).forEach(([id, newTest]) => {
+        const oldTest = mochaTestsById.get(id);
+        mochaTestsById.delete(id);
+        if (oldTest) removeTestOrSuite(oldTest);
+        if (newTest) mochaTestsById.set(id, addTest(rootSuite, newTest));
+      });
+    },
+  );
+
+  Object.values(tests)
+    .filter(isDefined)
+    .forEach((test) => mochaTestsById.set(test.id, addTest(rootSuite, test)));
+}

package/src/server/worker/index.ts

@@ -0,0 +1 @@
+export { start } from './worker.js';

package/src/server/worker/reporter.ts

@@ -0,0 +1,170 @@
+import chalk from 'chalk';
+import Logger from 'loglevel';
+import prefix from 'loglevel-plugin-prefix';
+import { Runner, reporters, MochaOptions } from 'mocha';
+import { Images, isDefined, isImageError } from '../../types.js';
+
+interface ReporterOptions {
+  reportDir: string;
+  sessionId: string;
+  topLevelSuite: string;
+  willRetry: boolean;
+  images: Partial<Record<string, Partial<Images>>>;
+}
+
+const testLevels: Record<string, string> = {
+  INFO: chalk.green('PASS'),
+  WARN: chalk.yellow('START'),
+  ERROR: chalk.red('FAIL'),
+};
+
+export class CreeveyReporter extends reporters.Base {
+  // TODO Output in better way, like vitest, maybe
+  constructor(runner: Runner, options: MochaOptions) {
+    super(runner);
+
+    const { sessionId, topLevelSuite } = options.reporterOptions as ReporterOptions;
+    const testLogger = Logger.getLogger(topLevelSuite);
+
+    prefix.apply(testLogger, {
+      format(level) {
+        return `${testLevels[level]} => (${topLevelSuite}:${chalk.gray(sessionId)})`;
+      },
+    });
+
+    runner.on('test', (test) => {
+      testLogger.warn(chalk.cyan(test.titlePath().join('/')));
+    });
+    runner.on('pass', (test) => {
+      testLogger.info(chalk.cyan(test.titlePath().join('/')));
+    });
+    runner.on('fail', (test, error) => {
+      testLogger.error(
+        chalk.cyan(test.titlePath().join('/')),
+        '\n  ',
+        getErrors(
+          error,
+          (error, imageName) => `${chalk.bold(imageName ?? topLevelSuite)}:${error}`,
+          (error) => error.stack ?? error.message,
+        ).join('\n  '),
+      );
+    });
+  }
+}
+
+export class TeamcityReporter extends reporters.Base {
+  constructor(runner: Runner, options: MochaOptions) {
+    super(runner);
+
+    const topLevelSuite = this.escape((options.reporterOptions as ReporterOptions).topLevelSuite);
+    const reporterOptions = options.reporterOptions as ReporterOptions;
+
+    runner.on('suite', (suite) => {
+      if (suite.root) console.log(`##teamcity[testSuiteStarted name='${topLevelSuite}' flowId='${process.pid}']`);
+      else console.log(`##teamcity[testSuiteStarted name='${this.escape(suite.title)}' flowId='${process.pid}']`);
+    });
+
+    runner.on('test', (test) => {
+      console.log(`##teamcity[testStarted name='${this.escape(test.title)}' flowId='${process.pid}']`);
+    });
+
+    runner.on('fail', (test, error: Error) => {
+      Object.entries(reporterOptions.images).forEach(([name, image]) => {
+        if (!image) return;
+        const filePath = test
+          .titlePath()
+          .concat(name == topLevelSuite ? [] : [topLevelSuite])
+          .map(this.escape)
+          .join('/');
+
+        const { error: _, ...rest } = image;
+        Object.values(rest as Partial<Images>)
+          .filter(isDefined)
+          .forEach((fileName) => {
+            console.log(
+              `##teamcity[publishArtifacts '${reporterOptions.reportDir}/${filePath}/${fileName} => report/${filePath}']`,
+            );
+            console.log(
+              `##teamcity[testMetadata testName='${this.escape(
+                test.title,
+              )}' type='image' value='report/${filePath}/${fileName}' flowId='${process.pid}']`,
+            );
+          });
+      });
+
+      // Output failed test as passed due TC don't support retry mechanic
+      // https://teamcity-support.jetbrains.com/hc/en-us/community/posts/207216829-Count-test-as-successful-if-at-least-one-try-is-successful?page=1#community_comment_207394125
+
+      if (reporterOptions.willRetry)
+        console.log(`##teamcity[testFinished name='${this.escape(test.title)}' flowId='${process.pid}']`);
+      else
+        console.log(
+          `##teamcity[testFailed name='${this.escape(test.title)}' message='${this.escape(
+            error.message,
+          )}' details='${this.escape(error.stack ?? '')}' flowId='${process.pid}']`,
+        );
+    });
+
+    runner.on('pending', (test) => {
+      console.log(
+        `##teamcity[testIgnored name='${this.escape(test.title)}' message='${this.escape(
+          typeof test.skipReason == 'boolean' ? test.title : test.skipReason,
+        )}' flowId='${process.pid}']`,
+      );
+    });
+
+    runner.on('test end', (test) => {
+      console.log(`##teamcity[testFinished name='${this.escape(test.title)}' flowId='${process.pid}']`);
+    });
+
+    runner.on('suite end', (suite) => {
+      if (!suite.root)
+        console.log(`##teamcity[testSuiteFinished name='${this.escape(suite.title)}' flowId='${process.pid}']`);
+    });
+
+    runner.on('end', () => {
+      console.log(`##teamcity[testSuiteFinished name='${topLevelSuite}' flowId='${process.pid}']`);
+    });
+  }
+
+  private escape = (str: string): string => {
+    if (!str) return '';
+    return (
+      str
+        .toString()
+        // eslint-disable-next-line no-control-regex
+        .replace(/\x1B.*?m/g, '')
+        .replace(/\|/g, '||')
+        .replace(/\n/g, '|n')
+        .replace(/\r/g, '|r')
+        .replace(/\[/g, '|[')
+        .replace(/\]/g, '|]')
+        .replace(/\u0085/g, '|x')
+        .replace(/\u2028/g, '|l')
+        .replace(/\u2029/g, '|p')
+        .replace(/'/g, "|'")
+    );
+  };
+}
+
+function getErrors(
+  error: unknown,
+  imageErrorToString: (error: string, imageName?: string) => string,
+  errorToString: (error: Error) => string,
+): string[] {
+  const errors = [];
+  if (!(error instanceof Error)) {
+    errors.push(error as string);
+  } else if (!isImageError(error)) {
+    errors.push(errorToString(error));
+  } else if (typeof error.images == 'string') {
+    errors.push(imageErrorToString(error.images));
+  } else {
+    const imageErrors = error.images;
+    Object.keys(imageErrors).forEach((imageName) => {
+      errors.push(imageErrorToString(imageErrors[imageName] ?? '', imageName));
+    });
+  }
+
+  return errors;
+}