@ohos-ports/web-test-runner 1.0.0-beta.0

package/src/config/collectGroupConfigs.ts

@@ -0,0 +1,50 @@
+import { ConfigLoaderError, readConfig } from '@web/config-loader';
+import { type TestRunnerGroupConfig } from '@web/test-runner-core';
+import globby from 'globby';
+import { TestRunnerStartError } from '../TestRunnerStartError.js';
+
+function validateGroupConfig(configFilePath: string, config: Partial<TestRunnerGroupConfig>) {
+  if (config.browsers != null && !Array.isArray(config.browsers)) {
+    throw new TestRunnerStartError(
+      `Group config at ${configFilePath} has invalid browsers option. It should be an array.`,
+    );
+  }
+
+  if (config.files != null && !(typeof config.files === 'string' || Array.isArray(config.files))) {
+    throw new TestRunnerStartError(
+      `Group config at ${configFilePath} has an invalid files option. It should be a string or an array.`,
+    );
+  }
+
+  return { name: configFilePath, configFilePath, ...config } as TestRunnerGroupConfig;
+}
+
+export async function collectGroupConfigs(patterns: string[]) {
+  const groupConfigFiles = new Set<string>();
+  const groupConfigs: TestRunnerGroupConfig[] = [];
+
+  for (const pattern of patterns) {
+    const filePaths = globby.sync(pattern, { absolute: true });
+    for (const filePath of filePaths) {
+      groupConfigFiles.add(filePath);
+    }
+  }
+
+  for (const groupConfigFile of groupConfigFiles) {
+    try {
+      const maybeGroupConfig = (await readConfig(
+        '',
+        groupConfigFile,
+      )) as Partial<TestRunnerGroupConfig>;
+      const groupConfig = validateGroupConfig(groupConfigFile, maybeGroupConfig);
+      groupConfigs.push(groupConfig);
+    } catch (error) {
+      if (error instanceof ConfigLoaderError) {
+        throw new TestRunnerStartError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  return groupConfigs;
+}

package/src/config/loadLauncher.ts

@@ -0,0 +1,48 @@
+import { TestRunnerStartError } from '../TestRunnerStartError.js';
+
+const puppeteerBrowsers = ['chrome', 'firefox'];
+const playwrightBrowsers = ['chromium', 'firefox', 'webkit'];
+
+function loadLauncher(name: string) {
+  const pkg = `@web/test-runner-${name}`;
+  try {
+    const path = require.resolve(pkg, { paths: [__dirname, process.cwd()] });
+    return require(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'MODULE_NOT_FOUND') {
+      throw new TestRunnerStartError(
+        `You need to add ${pkg} as a dependency of your project to use the --${name} flag.`,
+      );
+    } else {
+      throw error;
+    }
+  }
+}
+
+export function puppeteerLauncher(browsers: string[] = ['chrome']) {
+  for (const browser of browsers) {
+    if (!puppeteerBrowsers.includes(browser)) {
+      throw new TestRunnerStartError(
+        `Unknown puppeteer browser: ${browser}. ` +
+          `Supported browsers: ${puppeteerBrowsers.join(', ')}`,
+      );
+    }
+  }
+
+  const launcher = loadLauncher('puppeteer').puppeteerLauncher;
+  return browsers.map(product => launcher({ launchOptions: { product } }));
+}
+
+export function playwrightLauncher(browsers: string[] = ['chromium']) {
+  for (const browser of browsers) {
+    if (!playwrightBrowsers.includes(browser)) {
+      throw new TestRunnerStartError(
+        `Unknown playwright browser: ${browser}. ` +
+          `Supported browsers: ${playwrightBrowsers.join(', ')}`,
+      );
+    }
+  }
+
+  const launcher = loadLauncher('playwright').playwrightLauncher;
+  return browsers.map(product => launcher({ product }));
+}

package/src/config/mergeConfigs.ts

@@ -0,0 +1,26 @@
+import { type TestRunnerConfig } from './TestRunnerConfig.js';
+
+const arrayKeys = ['plugins', 'middleware'];
+
+export function mergeConfigs(...configs: Partial<TestRunnerConfig | undefined>[]) {
+  const finalConfig: any = {
+    plugins: [],
+    middleware: [],
+  };
+
+  for (const config of configs) {
+    if (config) {
+      for (const [key, value] of Object.entries(config)) {
+        if (arrayKeys.includes(key)) {
+          if (Array.isArray(value)) {
+            finalConfig[key].push(...value);
+          }
+        } else {
+          finalConfig[key] = value;
+        }
+      }
+    }
+  }
+
+  return finalConfig as Partial<TestRunnerConfig>;
+}

package/src/config/parseConfig.ts

@@ -0,0 +1,286 @@
+import { chromeLauncher } from '@web/test-runner-chrome';
+import {
+  emulateMediaPlugin,
+  filePlugin,
+  selectOptionPlugin,
+  sendKeysPlugin,
+  sendMousePlugin,
+  setUserAgentPlugin,
+  setViewportPlugin,
+  snapshotPlugin,
+} from '@web/test-runner-commands/plugins';
+import {
+  type CoverageConfig,
+  type TestRunnerCoreConfig,
+  type TestRunnerGroupConfig,
+} from '@web/test-runner-core';
+import { cpus } from 'os';
+import path from 'path';
+import { getPortPromise } from 'portfinder';
+
+import { esbuildPlugin, nodeResolvePlugin } from '@web/dev-server';
+import { TestRunnerStartError } from '../TestRunnerStartError.js';
+import { TestRunnerLogger } from '../logger/TestRunnerLogger.js';
+import { defaultReporter } from '../reporter/defaultReporter.js';
+import { type TestRunnerConfig } from './TestRunnerConfig.js';
+import { collectGroupConfigs } from './collectGroupConfigs.js';
+import { playwrightLauncher, puppeteerLauncher } from './loadLauncher.js';
+import { mergeConfigs } from './mergeConfigs.js';
+import { type TestRunnerCliArgs } from './readCliArgs.js';
+
+const secondMs = 1000;
+const minuteMs = secondMs * 60;
+
+const defaultConfig: Partial<TestRunnerConfig> = {
+  rootDir: process.cwd(),
+  protocol: 'http:',
+  hostname: 'localhost',
+  middleware: [],
+  plugins: [],
+  watch: false,
+  concurrentBrowsers: 2,
+  concurrency: Math.max(1, cpus().length / 2),
+  browserStartTimeout: minuteMs / 2,
+  testsStartTimeout: secondMs * 20,
+  testsFinishTimeout: minuteMs * 2,
+  browserLogs: true,
+};
+
+const defaultCoverageConfig: CoverageConfig = {
+  exclude: ['**/node_modules/**/*', '**/web_modules/**/*'],
+  threshold: { statements: 0, functions: 0, branches: 0, lines: 0 },
+  report: true,
+  reportDir: 'coverage',
+  reporters: ['lcov'],
+};
+
+function validate(config: Record<string, unknown>, key: string, type: string) {
+  if (config[key] == null) {
+    return;
+  }
+
+  if (typeof config[key] !== type) {
+    throw new TestRunnerStartError(`Configuration error: The ${key} setting should be a ${type}.`);
+  }
+}
+
+const stringSettings = ['rootDir', 'hostname'];
+const numberSettings = [
+  'port',
+  'concurrentBrowsers',
+  'concurrency',
+  'browserStartTimeout',
+  'testsStartTimeout',
+  'testsFinishTimeout',
+];
+const booleanSettings = [
+  'watch',
+  'preserveSymlinks',
+  'browserLogs',
+  'coverage',
+  'staticLogging',
+  'manual',
+  'open',
+  'debug',
+];
+
+export function validateConfig(config: Partial<TestRunnerConfig>) {
+  stringSettings.forEach(key => validate(config, key, 'string'));
+  numberSettings.forEach(key => validate(config, key, 'number'));
+  booleanSettings.forEach(key => validate(config, key, 'boolean'));
+
+  if (
+    config.esbuildTarget != null &&
+    !(typeof config.esbuildTarget === 'string' || Array.isArray(config.esbuildTarget))
+  ) {
+    throw new TestRunnerStartError(
+      `Configuration error: The esbuildTarget setting should be a string or array.`,
+    );
+  }
+
+  if (config.files != null && !(typeof config.files === 'string' || Array.isArray(config.files))) {
+    throw new TestRunnerStartError(
+      `Configuration error: The files setting should be a string or an array.`,
+    );
+  }
+
+  return config as TestRunnerConfig;
+}
+
+async function parseConfigGroups(config: TestRunnerConfig, cliArgs: TestRunnerCliArgs) {
+  const groupConfigs: TestRunnerGroupConfig[] = [];
+  const configPatterns: string[] = [];
+
+  if (cliArgs.groups) {
+    // groups are provided from CLI args
+    configPatterns.push(cliArgs.groups);
+  } else if (config.groups) {
+    // groups are provided from config
+    for (const entry of config.groups) {
+      if (typeof entry === 'object') {
+        groupConfigs.push(entry);
+      } else {
+        configPatterns.push(entry);
+      }
+    }
+  }
+
+  // group entries which are strings are globs which point to group conigs
+  groupConfigs.push(...(await collectGroupConfigs(configPatterns)));
+  return groupConfigs;
+}
+
+export async function parseConfig(
+  config: Partial<TestRunnerConfig>,
+  cliArgs: TestRunnerCliArgs = {},
+): Promise<{ config: TestRunnerCoreConfig; groupConfigs: TestRunnerGroupConfig[] }> {
+  const cliArgsConfig: Partial<TestRunnerConfig> = {
+    ...(cliArgs as Omit<TestRunnerCliArgs, 'groups' | 'browsers'>),
+  };
+  // CLI has properties with the same name as the config, but with different values
+  // delete them so they don't overwrite when merging, the CLI values are read separately
+  delete cliArgsConfig.groups;
+  delete cliArgsConfig.browsers;
+
+  const mergedConfigs = mergeConfigs(defaultConfig, config, cliArgsConfig);
+
+  // backwards compatibility for configs written for es-dev-server, where middleware was
+  // spelled incorrectly as middlewares
+  if (Array.isArray((mergedConfigs as any).middlewares)) {
+    mergedConfigs.middleware!.push(...(mergedConfigs as any).middlewares);
+  }
+
+  const finalConfig = validateConfig(mergedConfigs);
+  // filter out non-objects from plugin list
+  finalConfig.plugins = (finalConfig.plugins ?? []).filter(pl => typeof pl === 'object');
+
+  // ensure rootDir is always resolved
+  if (typeof finalConfig.rootDir === 'string') {
+    finalConfig.rootDir = path.resolve(finalConfig.rootDir);
+  } else {
+    throw new TestRunnerStartError('No rootDir specified.');
+  }
+
+  // generate a default random port
+  if (typeof finalConfig.port !== 'number') {
+    finalConfig.port = await getPortPromise({ port: 8000 });
+  }
+
+  finalConfig.coverageConfig = {
+    ...defaultCoverageConfig,
+    ...finalConfig.coverageConfig,
+  };
+
+  let groupConfigs = await parseConfigGroups(finalConfig, cliArgs);
+  if (groupConfigs.find(g => g.name === 'default')) {
+    throw new TestRunnerStartError(
+      'Cannot create a group named "default". This named is reserved by the test runner.',
+    );
+  }
+
+  if (cliArgs.group != null) {
+    if (cliArgs.group === 'default') {
+      // default group is an alias for the root config
+      groupConfigs = [];
+    } else {
+      const groupConfig = groupConfigs.find(c => c.name === cliArgs.group);
+      if (!groupConfig) {
+        throw new TestRunnerStartError(`Could not find any group named ${cliArgs.group}`);
+      }
+
+      // when focusing a group, ensure that the "default" group isn't run
+      // we can improve this by relying only on groups inside the test runner itself
+      if (groupConfig.files == null) {
+        groupConfig.files = finalConfig.files;
+      }
+      finalConfig.files = undefined;
+
+      groupConfigs = [groupConfig];
+    }
+  }
+
+  if (cliArgs.puppeteer) {
+    if (finalConfig.browsers && finalConfig.browsers.length > 0) {
+      throw new TestRunnerStartError(
+        'The --puppeteer flag cannot be used when defining browsers manually in your finalConfig.',
+      );
+    }
+    finalConfig.browsers = puppeteerLauncher(cliArgs.browsers);
+  } else if (cliArgs.playwright) {
+    if (finalConfig.browsers && finalConfig.browsers.length > 0) {
+      throw new TestRunnerStartError(
+        'The --playwright flag cannot be used when defining browsers manually in your finalConfig.',
+      );
+    }
+    finalConfig.browsers = playwrightLauncher(cliArgs.browsers);
+  } else {
+    if (cliArgs.browsers != null) {
+      throw new TestRunnerStartError(
+        `The browsers option must be used along with the puppeteer or playwright option.`,
+      );
+    }
+
+    // add default chrome launcher if the user did not configure their own browsers
+    if (!finalConfig.browsers) {
+      finalConfig.browsers = [chromeLauncher()];
+    }
+  }
+
+  finalConfig.testFramework = {
+    path: require.resolve('@web/test-runner-mocha/dist/autorun.js'),
+    ...(finalConfig.testFramework ?? {}),
+  };
+
+  if (!finalConfig.reporters) {
+    finalConfig.reporters = [defaultReporter()];
+  }
+
+  if (!finalConfig.logger) {
+    finalConfig.logger = new TestRunnerLogger(config.debug);
+  }
+
+  if (finalConfig.plugins == null) {
+    finalConfig.plugins = [];
+  }
+
+  // plugin with a noop transformImport hook, this will cause the dev server to analyze modules and
+  // catch syntax errors. this way we still report syntax errors when the user has no flags enabled
+  finalConfig.plugins.unshift({
+    name: 'syntax-checker',
+    transformImport() {
+      return undefined;
+    },
+  });
+
+  finalConfig.plugins.unshift(
+    setViewportPlugin(),
+    emulateMediaPlugin(),
+    setUserAgentPlugin(),
+    selectOptionPlugin(),
+    filePlugin(),
+    sendKeysPlugin(),
+    sendMousePlugin(),
+    snapshotPlugin({ updateSnapshots: !!cliArgs.updateSnapshots }),
+  );
+
+  if (finalConfig.nodeResolve) {
+    const userOptions =
+      typeof finalConfig.nodeResolve === 'object' ? finalConfig.nodeResolve : undefined;
+    // do node resolve after user plugins, to allow user plugins to resolve imports
+    finalConfig.plugins!.push(
+      nodeResolvePlugin(finalConfig.rootDir!, finalConfig.preserveSymlinks, userOptions),
+    );
+  }
+
+  if (finalConfig?.esbuildTarget) {
+    finalConfig.plugins!.unshift(esbuildPlugin(finalConfig.esbuildTarget));
+  }
+
+  if ((!finalConfig.files || finalConfig.files.length === 0) && groupConfigs.length === 0) {
+    throw new TestRunnerStartError(
+      'Did not find any tests to run. Use the "files" or "groups" option to configure test files.',
+    );
+  }
+
+  return { config: finalConfig as TestRunnerCoreConfig, groupConfigs };
+}

package/src/config/readCliArgs.ts

@@ -0,0 +1,187 @@
+import camelCase from 'camelcase';
+import commandLineArgs from 'command-line-args';
+import commandLineUsage, { OptionDefinition } from 'command-line-usage';
+
+import { type TestRunnerConfig } from './TestRunnerConfig.js';
+
+export interface TestRunnerCliArgs extends Partial<
+  Pick<
+    TestRunnerConfig,
+    | 'files'
+    | 'rootDir'
+    | 'watch'
+    | 'coverage'
+    | 'concurrentBrowsers'
+    | 'concurrency'
+    | 'staticLogging'
+    | 'manual'
+    | 'open'
+    | 'port'
+    | 'preserveSymlinks'
+    | 'nodeResolve'
+    | 'debug'
+    | 'esbuildTarget'
+  >
+> {
+  config?: string;
+  groups?: string;
+  group?: string;
+  puppeteer?: boolean;
+  playwright?: boolean;
+  browsers?: string[];
+  updateSnapshots?: boolean;
+}
+
+const options: OptionDefinition[] = [
+  {
+    name: 'files',
+    type: String,
+    multiple: true,
+    defaultOption: true,
+    description: 'Test files to run',
+  },
+  {
+    name: 'root-dir',
+    type: String,
+    description: 'Root directory to serve files from.',
+  },
+  {
+    name: 'watch',
+    type: Boolean,
+    description: 'Reload tests on file changes',
+  },
+  {
+    name: 'coverage',
+    type: Boolean,
+    description: 'Check for code coverage. Slows down testing.',
+  },
+  {
+    name: 'concurrent-browsers',
+    type: Number,
+    description: 'Amount of browsers to run concurrently. Defaults to 2.',
+  },
+  {
+    name: 'concurrency',
+    type: Number,
+    description:
+      'Amount of test files to run concurrently. Defaults to total CPU cores divided by 2.',
+  },
+  {
+    name: 'config',
+    type: String,
+    description: 'Location to read config file from.',
+  },
+  {
+    name: 'static-logging',
+    type: Boolean,
+    description: 'Disables rendering a progress bar dynamically to the terminal.',
+  },
+  {
+    name: 'manual',
+    type: Boolean,
+    description:
+      'Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL.',
+  },
+  {
+    name: 'open',
+    type: Boolean,
+    description: 'Opens browser for manual testing. Requires the manual option to be set.',
+  },
+  {
+    name: 'port',
+    type: Number,
+    description: 'Port to bind the server on.',
+  },
+  {
+    name: 'groups',
+    type: String,
+    description: 'Pattern of group config files.',
+  },
+  {
+    name: 'group',
+    type: String,
+    description:
+      'Name of the group to run tests for. When this is set, the other groups are ignored.',
+  },
+  {
+    name: 'preserve-symlinks',
+    type: Boolean,
+    description: "Don't follow symlinks when resolving imports",
+  },
+  {
+    name: 'puppeteer',
+    type: Boolean,
+    description: 'Run tests using puppeteer',
+  },
+  {
+    name: 'playwright',
+    type: Boolean,
+    description: 'Run tests using playwright',
+  },
+  {
+    name: 'browsers',
+    type: String,
+    multiple: true,
+    description: 'Browsers to run when choosing puppeteer or playwright',
+  },
+  {
+    name: 'node-resolve',
+    type: Boolean,
+    description: 'Resolve bare module imports using node resolution',
+  },
+  {
+    name: 'update-snapshots',
+    type: Boolean,
+    description: 'Whether to accept changes in shapshots, and save them on disk',
+  },
+  {
+    name: 'esbuild-target',
+    type: String,
+    multiple: true,
+    description:
+      'JS language target to compile down to using esbuild. Recommended value is "auto", which compiles based on user agent. Check the docs for more options.',
+  },
+  {
+    name: 'debug',
+    type: Boolean,
+    description: 'Log debug messages',
+  },
+  {
+    name: 'help',
+    type: Boolean,
+    description: 'Print help commands',
+  },
+];
+
+export interface ReadCliArgsParams {
+  argv?: string[];
+}
+
+export function readCliArgs({ argv = process.argv }: ReadCliArgsParams = {}): TestRunnerCliArgs {
+  const cliArgs = commandLineArgs(options, { argv, partial: true });
+
+  if ('help' in cliArgs) {
+    console.log(
+      commandLineUsage([
+        {
+          header: 'Web Test Runner',
+          content: 'Test runner for web applications.',
+        },
+        {
+          header: 'Usage',
+          content: 'web-test-runner [options...]' + '\nwtr [options...]',
+        },
+        { header: 'Options', optionList: options },
+      ]),
+    );
+    process.exit();
+  }
+
+  const cliArgsConfig: TestRunnerCliArgs = {};
+
+  for (const [key, value] of Object.entries(cliArgs)) {
+    cliArgsConfig[camelCase(key) as keyof TestRunnerCliArgs] = value;
+  }
+
+  return cliArgsConfig;
+}

package/src/config/readFileConfig.ts

@@ -0,0 +1,35 @@
+import { ConfigLoaderError, readConfig } from '@web/config-loader';
+
+import { TestRunnerStartError } from '../TestRunnerStartError.js';
+
+export interface ReadFileConfigParams {
+  /**
+   * The config name to use. Defaults to web-test-runner.config.{mjs,cjs,js}
+   */
+  configName?: string;
+  /**
+   * Path to the config file. If this is not set, the config is looked up in the
+   * current working directory using the config name.
+   */
+  configPath?: string;
+}
+
+/**
+ * Reads the config from disk, defaults to process.cwd() + web-test-runner.config.{mjs,js,cjs} or
+ * a custom config path.
+ * @param customConfig the custom location to read the config from
+ */
+
+export async function readFileConfig({
+  configName = 'web-test-runner.config',
+  configPath,
+}: ReadFileConfigParams = {}) {
+  try {
+    return await readConfig(configName, typeof configPath === 'string' ? configPath : undefined);
+  } catch (error) {
+    if (error instanceof ConfigLoaderError) {
+      throw new TestRunnerStartError(error.message);
+    }
+    throw error;
+  }
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+import type { TestRunnerConfig as FullTestRunnerConfig } from './config/TestRunnerConfig.js';
+
+export { chromeLauncher } from '@web/test-runner-chrome';
+export * from '@web/test-runner-core';
+export { defaultReporter } from './reporter/defaultReporter.js';
+export { dotReporter } from './reporter/dotReporter.js';
+export { formatError } from './reporter/reportTestsErrors.js';
+export { summaryReporter } from './reporter/summaryReporter.js';
+export { startTestRunner } from './startTestRunner.js';
+
+export type TestRunnerConfig = Partial<FullTestRunnerConfig>;

package/src/logger/TestRunnerLogger.ts

@@ -0,0 +1,54 @@
+import { type ErrorWithLocation, type Logger } from '@web/test-runner-core';
+
+export class TestRunnerLogger implements Logger {
+  loggedSyntaxErrors = new Map<string, ErrorWithLocation[]>();
+
+  constructor(private debugLogging: boolean = false) {}
+
+  log(...messages: unknown[]) {
+    console.log(...messages);
+  }
+
+  debug(...messages: unknown[]) {
+    if (this.debugLogging) {
+      console.debug(...messages);
+    }
+  }
+
+  error(...messages: unknown[]) {
+    console.error(...messages);
+  }
+
+  warn(...messages: unknown[]) {
+    console.warn(...messages);
+  }
+
+  group() {
+    console.group();
+  }
+
+  groupEnd() {
+    console.groupEnd();
+  }
+
+  logSyntaxError(error: ErrorWithLocation) {
+    const { message, code, filePath, column, line } = error;
+    let errors = this.loggedSyntaxErrors.get(filePath);
+    if (!errors) {
+      errors = [];
+      this.loggedSyntaxErrors.set(filePath, errors);
+    } else if (
+      errors.find(
+        e => e.code === code && e.message === message && e.column === column && e.line === line,
+      )
+    ) {
+      // dedupe syntax errors we already logged
+      return;
+    }
+    errors.push(error);
+  }
+
+  clearLoggedSyntaxErrors() {
+    this.loggedSyntaxErrors = new Map();
+  }
+}