creevey 0.10.0-beta.4 → 0.10.0-beta.41

package/src/server/providers/browser.ts

@@ -5,6 +5,7 @@ import { isDefined } from '../../types.js';
 import { logger } from '../logger.js';
 import { deserializeRawStories } from '../../shared/index.js';
 
+// TODO Don't have updates from stories
 export const loadStories: StoriesProvider = async (_config, storiesListener, webdriver) => {
   if (cluster.isPrimary) {
     return new Promise<StoriesRaw>((resolve) => {
@@ -17,13 +18,13 @@ export const loadStories: StoriesProvider = async (_config, storiesListener, web
           if (message.type == 'set') {
             const { stories, oldTests } = message.payload;
             if (oldTests.length > 0)
-              logger.warn(
+              logger().warn(
                 `If you use browser stories provider of CSFv3 Storybook feature\n` +
                   `Creevey will not load tests defined in story parameters from following stories:\n` +
                   oldTests.join('\n'),
               );
             unsubscribe();
-            resolve(stories);
+            resolve(deserializeRawStories(stories));
           }
         });
         sendStoriesMessage(worker, { type: 'get' });
@@ -36,10 +37,11 @@ export const loadStories: StoriesProvider = async (_config, storiesListener, web
   } else {
     subscribeOn('stories', (message) => {
       if (message.type == 'get')
-        emitStoriesMessage({ type: 'set', payload: { stories, oldTests: storiesWithOldTests } });
+        emitStoriesMessage({ type: 'set', payload: { stories: rawStories, oldTests: storiesWithOldTests } });
       if (message.type == 'update') storiesListener(new Map(message.payload));
     });
-    const stories = deserializeRawStories((await webdriver?.loadStoriesFromBrowser()) ?? {});
+    const rawStories = (await webdriver?.loadStoriesFromBrowser()) ?? {};
+    const stories = deserializeRawStories(rawStories);
 
     const storiesWithOldTests: string[] = [];
 

package/src/server/providers/hybrid.ts

@@ -54,7 +54,7 @@ async function parseParams(
 
   if (listener) {
     chokidar.watch(testFiles).on('change', (filePath) => {
-      logger.debug(`changed: ${filePath}`);
+      logger().debug(`changed: ${filePath}`);
 
       // doesn't work, always returns {} due modules caching
       // see https://github.com/nodejs/modules/issues/307

package/src/server/reporters/creevey.ts

@@ -0,0 +1,71 @@
+import chalk from 'chalk';
+import Logger from 'loglevel';
+import prefix from 'loglevel-plugin-prefix';
+import { FakeTest, isImageError, TEST_EVENTS } from '../../types.js';
+import EventEmitter from 'events';
+
+const testLevels: Record<string, string> = {
+  INFO: chalk.green('PASS'),
+  WARN: chalk.yellow('START'),
+  ERROR: chalk.red('FAIL'),
+};
+
+export class CreeveyReporter {
+  private logger: Logger.Logger | null = null;
+  // TODO Output in better way, like vitest, maybe
+  constructor(runner: EventEmitter) {
+    runner.on(TEST_EVENTS.TEST_BEGIN, (test: FakeTest) => {
+      this.getLogger(test.creevey).warn(chalk.cyan(test.fullTitle()));
+    });
+    runner.on(TEST_EVENTS.TEST_PASS, (test: FakeTest) => {
+      this.getLogger(test.creevey).info(chalk.cyan(test.fullTitle()), chalk.gray(`(${test.duration} ms)`));
+    });
+    runner.on(TEST_EVENTS.TEST_FAIL, (test: FakeTest, error) => {
+      this.getLogger(test.creevey).error(
+        chalk.cyan(test.fullTitle()),
+        chalk.gray(`(${test.duration} ms)`),
+        '\n  ',
+        this.getErrors(
+          error,
+          (error, imageName) => `${chalk.bold(imageName ?? test.creevey.browserName)}:${error}`,
+          (error) => error.stack ?? error.message,
+        ).join('\n  '),
+      );
+    });
+  }
+
+  private getLogger(options: { sessionId: string; browserName: string }) {
+    if (this.logger) return this.logger;
+    const { sessionId, browserName } = options;
+    const testLogger = Logger.getLogger(sessionId);
+
+    this.logger = prefix.apply(testLogger, {
+      format(level) {
+        return `[${browserName}:${chalk.gray(process.pid)}] ${testLevels[level]} => ${chalk.gray(sessionId)}`;
+      },
+    });
+
+    return this.logger;
+  }
+
+  private getErrors(
+    error: unknown,
+    imageErrorToString: (error: string, imageName?: string) => string,
+    errorToString: (error: Error) => string,
+  ): string[] {
+    const errors = [];
+    if (!(error instanceof Error)) {
+      errors.push(error as string);
+    } else if (!isImageError(error)) {
+      errors.push(errorToString(error));
+    } else if (typeof error.images == 'string') {
+      errors.push(imageErrorToString(error.images));
+    } else {
+      const imageErrors = error.images ?? {};
+      Object.keys(imageErrors).forEach((imageName) => {
+        errors.push(imageErrorToString(imageErrors[imageName] ?? '', imageName));
+      });
+    }
+    return errors;
+  }
+}

package/src/server/reporters/index.ts

@@ -0,0 +1,11 @@
+import { BaseReporter } from '../../types.js';
+import { CreeveyReporter } from './creevey.js';
+import { JUnitReporter } from './junit.js';
+import { TeamcityReporter } from './teamcity.js';
+
+export function getReporter(reporter: BaseReporter | 'creevey' | 'teamcity' | 'junit'): BaseReporter {
+  if (reporter === 'creevey') return CreeveyReporter;
+  if (reporter === 'teamcity') return TeamcityReporter;
+  if (reporter === 'junit') return JUnitReporter;
+  return reporter;
+}

package/src/server/reporters/junit.ts

@@ -0,0 +1,205 @@
+import EventEmitter from 'events';
+import { dirname, resolve } from 'path';
+import { closeSync, existsSync, mkdirSync, openSync, writeFileSync } from 'fs';
+import { TEST_EVENTS, FakeTest } from '../../types.js';
+import { logger } from '../logger.js';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+class IndentedLogger<T = any> {
+  private currentIndent = '';
+
+  constructor(private baseLog: (text: string) => T) {}
+
+  indent(): void {
+    this.currentIndent += '    ';
+  }
+
+  unindent(): void {
+    this.currentIndent = this.currentIndent.substring(0, this.currentIndent.length - 4);
+  }
+
+  log(text: string): T {
+    return this.baseLog(this.currentIndent + text);
+  }
+}
+
+// NOTE: This is a reworked copy of the JUnitReporter class from Vitest.
+export class JUnitReporter {
+  private reportFile: string;
+  private fileFd?: number;
+  private logger: IndentedLogger<void>;
+  private suites: Record<string, FakeTest[]> = {};
+  // TODO classnameTemplate
+  constructor(runner: EventEmitter, options: { reportDir: string; reporterOptions: { outputFile?: string } }) {
+    const { reportDir, reporterOptions } = options;
+
+    this.reportFile = reporterOptions.outputFile ?? resolve(reportDir, 'junit.xml');
+
+    this.logger = new IndentedLogger((text) => {
+      this.fileFd ??= openSync(this.reportFile, 'w+');
+
+      writeFileSync(this.fileFd, `${text}\n`);
+    });
+
+    runner.on(TEST_EVENTS.RUN_BEGIN, () => {
+      this.suites = {};
+
+      const outputDirectory = dirname(this.reportFile);
+      if (!existsSync(outputDirectory)) {
+        mkdirSync(outputDirectory, { recursive: true });
+      }
+
+      this.fileFd = openSync(this.reportFile, 'w+');
+    });
+    runner.on(TEST_EVENTS.TEST_PASS, (test: FakeTest) => {
+      const suite = this.suites[test.parent.title] ?? [];
+      suite.push(test);
+    });
+    runner.on(TEST_EVENTS.TEST_FAIL, (test: FakeTest) => {
+      const suite = this.suites[test.parent.title] ?? [];
+      suite.push(test);
+    });
+    runner.on(TEST_EVENTS.RUN_END, () => {
+      this.onFinished();
+    });
+  }
+
+  private writeElement(name: string, attrs: Record<string, string | number | undefined>, children?: () => void): void {
+    const pairs: string[] = [];
+    for (const key in attrs) {
+      const attr = attrs[key];
+      if (attr === undefined) {
+        continue;
+      }
+
+      pairs.push(`${key}="${escapeXML(attr)}"`);
+    }
+
+    this.logger.log(`<${name}${pairs.length ? ` ${pairs.join(' ')}` : ''}>`);
+    this.logger.indent();
+    children?.call(this);
+    this.logger.unindent();
+
+    this.logger.log(`</${name}>`);
+  }
+
+  private writeTasks(tests: FakeTest[]): void {
+    for (const test of tests) {
+      const classname = test.parent.title;
+
+      this.writeElement(
+        'testcase',
+        {
+          classname,
+          name: test.title,
+          time: getDuration(test),
+        },
+        () => {
+          if (test.state === 'failed') {
+            const error = test.err;
+            this.writeElement('failure', { message: error });
+          }
+        },
+      );
+    }
+  }
+
+  private onFinished(): void {
+    this.logger.log('<?xml version="1.0" encoding="UTF-8" ?>');
+
+    const suites = Object.entries(this.suites).map(([name, tests]) => {
+      return {
+        name,
+        tests,
+        failures: tests.filter((test) => test.state === 'failed').length,
+        time: tests.reduce((acc, test) => acc + (test.duration ?? 0), 0),
+      };
+    });
+    const stats = suites.reduce(
+      (s, { tests, failures, time }) => {
+        s.tests += tests.length;
+        s.failures += failures;
+        s.time += time;
+        return s;
+      },
+      { name: 'creevey tests', tests: 0, failures: 0, time: 0 },
+    );
+
+    this.writeElement('testsuites', { ...stats, time: executionTime(stats.time) }, () => {
+      suites.forEach(({ name, tests, failures, time }) => {
+        this.writeElement(
+          'testsuite',
+          {
+            name,
+            tests: tests.length,
+            failures,
+            time: executionTime(time),
+          },
+          () => {
+            this.writeTasks(tests);
+          },
+        );
+      });
+    });
+
+    if (this.reportFile) {
+      logger().info(`JUNIT report written to ${this.reportFile}`);
+    }
+
+    if (this.fileFd) {
+      closeSync(this.fileFd);
+      this.fileFd = undefined;
+    }
+  }
+}
+
+// https://gist.github.com/john-doherty/b9195065884cdbfd2017a4756e6409cc
+function removeInvalidXMLCharacters(value: string, removeDiscouragedChars: boolean): string {
+  let regex =
+    // eslint-disable-next-line no-control-regex
+    /([\0-\x08\v\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF])/g;
+  value = String(value).replace(regex, '');
+
+  if (removeDiscouragedChars) {
+    // remove everything discouraged by XML 1.0 specifications
+    regex = new RegExp(
+      '([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|\\uD83F[\\uDFFE\\uDFFF]|(?:\\uD87F[\\uDF' +
+        'FE\\uDFFF])|\\uD8BF[\\uDFFE\\uDFFF]|\\uD8FF[\\uDFFE\\uDFFF]|(?:\\uD93F[\\uDFFE\\uD' +
+        'FFF])|\\uD97F[\\uDFFE\\uDFFF]|\\uD9BF[\\uDFFE\\uDFFF]|\\uD9FF[\\uDFFE\\uDFFF]' +
+        '|\\uDA3F[\\uDFFE\\uDFFF]|\\uDA7F[\\uDFFE\\uDFFF]|\\uDABF[\\uDFFE\\uDFFF]|(?:\\' +
+        'uDAFF[\\uDFFE\\uDFFF])|\\uDB3F[\\uDFFE\\uDFFF]|\\uDB7F[\\uDFFE\\uDFFF]|(?:\\uDBBF' +
+        '[\\uDFFE\\uDFFF])|\\uDBFF[\\uDFFE\\uDFFF](?:[\\0-\\t\\v\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
+        'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
+        '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
+      'g',
+    );
+
+    value = value.replace(regex, '');
+  }
+
+  return value;
+}
+
+function escapeXML(value: string | number): string {
+  return removeInvalidXMLCharacters(
+    String(value)
+      .replace(/&/g, '&amp;')
+      .replace(/"/g, '&quot;')
+      .replace(/'/g, '&apos;')
+      .replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;'),
+    true,
+  );
+}
+
+function executionTime(durationMS: number) {
+  return (durationMS / 1000).toLocaleString('en-US', {
+    useGrouping: false,
+    maximumFractionDigits: 10,
+  });
+}
+
+function getDuration(task: FakeTest): string | undefined {
+  const duration = task.duration ?? 0;
+  return executionTime(duration);
+}

package/src/server/reporters/teamcity.ts

@@ -0,0 +1,74 @@
+import { FakeTest, Images, isDefined, TEST_EVENTS } from '../../types.js';
+import EventEmitter from 'events';
+
+export class TeamcityReporter {
+  constructor(runner: EventEmitter, options: { reportDir: string }) {
+    const { reportDir } = options;
+
+    runner.on(TEST_EVENTS.TEST_BEGIN, (test: FakeTest) => {
+      console.log(`##teamcity[testStarted name='${this.escape(test.fullTitle())}' flowId='${test.creevey.workerId}']`);
+    });
+
+    runner.on(TEST_EVENTS.TEST_PASS, (test: FakeTest) => {
+      console.log(`##teamcity[testFinished name='${this.escape(test.fullTitle())}' flowId='${test.creevey.workerId}']`);
+    });
+
+    runner.on(TEST_EVENTS.TEST_FAIL, (test: FakeTest, error: Error) => {
+      const browserName = this.escape(test.creevey.browserName);
+      Object.entries(test.creevey.images).forEach(([name, image]) => {
+        if (!image) return;
+        const filePath = test
+          .titlePath()
+          .slice(0, -1)
+          .concat(name == browserName ? [] : [browserName])
+          .map(this.escape)
+          .join('/');
+
+        const { error: _, ...rest } = image;
+        Object.values(rest as Partial<Images>)
+          .filter(isDefined)
+          .forEach((fileName) => {
+            console.log(`##teamcity[publishArtifacts '${reportDir}/${filePath}/${fileName} => report/${filePath}']`);
+            console.log(
+              `##teamcity[testMetadata testName='${this.escape(
+                test.fullTitle(),
+              )}' type='image' value='report/${filePath}/${fileName}' flowId='${test.creevey.workerId}']`,
+            );
+          });
+      });
+
+      // Output failed test as passed due TC don't support retry mechanic
+      // https://teamcity-support.jetbrains.com/hc/en-us/community/posts/207216829-Count-test-as-successful-if-at-least-one-try-is-successful?page=1#community_comment_207394125
+
+      if (test.creevey.willRetry)
+        console.log(
+          `##teamcity[testFinished name='${this.escape(test.fullTitle())}' flowId='${test.creevey.workerId}']`,
+        );
+      else
+        console.log(
+          `##teamcity[testFailed name='${this.escape(test.fullTitle())}' message='${this.escape(
+            error.message,
+          )}' details='${this.escape(error.stack ?? '')}' flowId='${test.creevey.workerId}']`,
+        );
+    });
+  }
+
+  private escape = (str: string): string => {
+    if (!str) return '';
+    return (
+      str
+        .toString()
+        // eslint-disable-next-line no-control-regex
+        .replace(/\x1B.*?m/g, '')
+        .replace(/\|/g, '||')
+        .replace(/\n/g, '|n')
+        .replace(/\r/g, '|r')
+        .replace(/\[/g, '|[')
+        .replace(/\]/g, '|]')
+        .replace(/\u0085/g, '|x')
+        .replace(/\u2028/g, '|l')
+        .replace(/\u2029/g, '|p')
+        .replace(/'/g, "|'")
+    );
+  };
+}