@jsenv/test 1.7.9 → 1.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/test",
-  "version": "1.7.9",
+  "version": "1.8.1",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -36,12 +36,12 @@
     "@jsenv/abort": "4.2.4",
     "@jsenv/ast": "6.0.1",
     "@jsenv/filesystem": "4.3.2",
-    "@jsenv/log": "3.4.2",
+    "@jsenv/log": "3.4.3",
     "@jsenv/plugin-supervisor": "1.3.9",
     "@jsenv/urls": "2.2.1",
     "@jsenv/url-meta": "8.1.0",
     "@jsenv/utils": "2.0.1",
-    "@jsenv/github-check-run": "0.1.3",
+    "@jsenv/github-check-run": "0.1.4",
     "v8-to-istanbul": "9.1.3",
     "istanbul-lib-coverage": "3.2.2",
     "istanbul-lib-instrument": "6.0.1",

package/src/execution/execute_test_plan.js

@@ -1,4 +1,6 @@
 import { existsSync } from "node:fs";
+import { memoryUsage } from "node:process";
+import { takeCoverage } from "node:v8";
 import stripAnsi from "strip-ansi";
 import { Abort, raceProcessTeardownEvents } from "@jsenv/abort";
 import { URL_META } from "@jsenv/url-meta";
@@ -7,22 +9,36 @@ import {
   ensureEmptyDirectory,
   assertAndNormalizeDirectoryUrl,
   assertAndNormalizeFileUrl,
+  writeFileSync,
 } from "@jsenv/filesystem";
-import { createLogger, createDetailedMessage, UNICODE } from "@jsenv/log";
+import {
+  createLogger,
+  createDetailedMessage,
+  UNICODE,
+  createLog,
+  startSpinner,
+} from "@jsenv/log";
 import {
   startGithubCheckRun,
   readGitHubWorkflowEnv,
 } from "@jsenv/github-check-run";
 
 import { createTeardown } from "../helpers/teardown.js";
+import { createCallOrderer } from "../helpers/call_orderer.js";
+import { reportToCoverage } from "../coverage/report_to_coverage.js";
 import { generateCoverageJsonFile } from "../coverage/coverage_reporter_json_file.js";
 import { generateCoverageHtmlDirectory } from "../coverage/coverage_reporter_html_directory.js";
 import { generateCoverageTextLog } from "../coverage/coverage_reporter_text_log.js";
 import { assertAndNormalizeWebServer } from "./web_server_param.js";
 import { executionStepsFromTestPlan } from "./execution_steps.js";
-import { executeSteps } from "./execute_steps.js";
-import { formatSummary } from "./logs_file_execution.js";
+import {
+  createExecutionLog,
+  formatSummaryLog,
+  formatSummary,
+} from "./logs_file_execution.js";
 import { githubAnnotationFromError } from "./github_annotation_from_error.js";
+import { run } from "./run.js";
+import { ensureGlobalGc } from "./gc.js";
 
 /**
  * Execute a list of files and log how it goes.
@@ -74,7 +90,7 @@ export const executeTestPlan = async ({
   githubCheckEnabled = Boolean(process.env.GITHUB_WORKFLOW),
   githubCheckLogLevel,
   githubCheckName = "Jsenv tests",
-  githubCheckTitle,
+  githubCheckTitle = "Tests executions",
   githubCheckToken,
   githubCheckRepositoryOwner,
   githubCheckRepositoryName,
@@ -119,10 +135,21 @@ export const executeTestPlan = async ({
   coverageReportJsonFileUrl,
   coverageReportHtml = !process.env.CI,
   coverageReportHtmlDirectoryUrl,
+
+  beforeExecutionCallback = () => {},
+  afterExecutionCallback = () => {},
+  afterAllExecutionCallback = () => {},
   ...rest
 }) => {
   const teardown = createTeardown();
 
+  const beforeExecutionCallbackSet = new Set();
+  const afterExecutionCallbackSet = new Set();
+  const afterAllExecutionCallbackSet = new Set();
+  beforeExecutionCallbackSet.add(beforeExecutionCallback);
+  afterExecutionCallbackSet.add(afterExecutionCallback);
+  afterAllExecutionCallbackSet.add(afterAllExecutionCallback);
+
   const operation = Abort.startOperation();
   operation.addAbortSignal(signal);
   if (handleSIGINT) {
@@ -362,15 +389,12 @@ To fix this warning:
     "**/.jsenv/": null,
   };
   logger.debug(`Generate executions`);
-  const executionSteps = await executionStepsFromTestPlan({
+  let executionSteps = await executionStepsFromTestPlan({
     signal,
     testPlan,
     rootDirectoryUrl,
   });
   logger.debug(`${executionSteps.length} executions planned`);
-  let beforeExecutionCallback;
-  let afterExecutionCallback;
-  let afterAllExecutionCallback = () => {};
   if (githubCheckEnabled) {
     const githubCheckRun = await startGithubCheckRun({
       logLevel: githubCheckLogLevel,
@@ -379,11 +403,11 @@ To fix this warning:
       repositoryName: githubCheckRepositoryName,
       commitSha: githubCheckCommitSha,
       checkName: githubCheckName,
-      checkTitle: `Tests executions`,
+      checkTitle: githubCheckTitle,
       checkSummary: `${executionSteps.length} files will be executed`,
     });
     const annotations = [];
-    afterExecutionCallback = (afterExecutionInfo) => {
+    afterExecutionCallbackSet.add((afterExecutionInfo) => {
       const { executionResult } = afterExecutionInfo;
       const { errors = [] } = executionResult;
       for (const error of errors) {
@@ -393,79 +417,338 @@ To fix this warning:
         });
         annotations.push(annotation);
       }
-    };
-    afterAllExecutionCallback = async ({ testPlanSummary }) => {
+    });
+    afterAllExecutionCallbackSet.add(async (returnValue) => {
+      const { summary } = returnValue;
       const title = "Jsenv test results";
-      const summary = stripAnsi(formatSummary(testPlanSummary));
-      if (
-        testPlanSummary.counters.total !== testPlanSummary.counters.completed
-      ) {
+      const summaryText = stripAnsi(formatSummary(summary));
+      if (summary.counters.total !== summary.counters.completed) {
         await githubCheckRun.fail({
           title,
-          summary,
+          summary: summaryText,
           annotations,
         });
         return;
       }
       await githubCheckRun.pass({
         title,
-        summary,
+        summary: summaryText,
         annotations,
       });
-    };
+    });
   }
 
-  const result = await executeSteps(executionSteps, {
-    signal,
-    teardown,
-    logger,
-    logRefresh,
-    logSummary,
-    logRuntime,
-    logEachDuration,
-    logTimeUsage,
-    logMemoryHeapUsage,
-    logFileRelativeUrl,
-    logShortForCompletedExecutions,
-    logMergeForCompletedExecutions,
-    rootDirectoryUrl,
-    webServer,
-
-    maxExecutionsInParallel,
-    defaultMsAllocatedPerExecution,
-    failFast,
-    keepRunning,
-    cooldownBetweenExecutions,
-    gcBetweenExecutions,
-
-    githubCheckEnabled,
-    githubCheckName,
-    githubCheckTitle,
-    githubCheckToken,
-    githubCheckRepositoryOwner,
-    githubCheckRepositoryName,
-    githubCheckCommitSha,
-
-    coverageEnabled,
-    coverageConfig,
-    coverageIncludeMissing,
-    coverageMethodForBrowsers,
-    coverageMethodForNodeJs,
-    coverageV8ConflictWarning,
-    coverageTempDirectoryUrl,
-
-    beforeExecutionCallback,
-    afterExecutionCallback,
-  });
+  executionSteps = executionSteps.filter(
+    (executionStep) => !executionStep.runtime?.disabled,
+  );
+
+  const returnValue = {
+    aborted: false,
+    summary: null,
+    report: null,
+    coverage: null,
+  };
+  const report = {};
+  const callbacks = [];
+
+  const multipleExecutionsOperation = Abort.startOperation();
+  multipleExecutionsOperation.addAbortSignal(signal);
+  const failFastAbortController = new AbortController();
+  if (failFast) {
+    multipleExecutionsOperation.addAbortSignal(failFastAbortController.signal);
+  }
+
+  try {
+    if (gcBetweenExecutions) {
+      ensureGlobalGc();
+    }
+
+    if (coverageEnabled) {
+      // when runned multiple times, we don't want to keep previous files in this directory
+      await ensureEmptyDirectory(coverageTempDirectoryUrl);
+      callbacks.push(async () => {
+        if (multipleExecutionsOperation.signal.aborted) {
+          // don't try to do the coverage stuff
+          return;
+        }
+        try {
+          if (coverageMethodForNodeJs === "NODE_V8_COVERAGE") {
+            takeCoverage();
+            // conceptually we don't need coverage anymore so it would be
+            // good to call v8.stopCoverage()
+            // but it logs a strange message about "result is not an object"
+          }
+          const coverage = await reportToCoverage(report, {
+            signal: multipleExecutionsOperation.signal,
+            logger,
+            rootDirectoryUrl,
+            coverageConfig,
+            coverageIncludeMissing,
+            coverageMethodForNodeJs,
+            coverageV8ConflictWarning,
+          });
+          returnValue.coverage = coverage;
+        } catch (e) {
+          if (Abort.isAbortError(e)) {
+            return;
+          }
+          throw e;
+        }
+      });
+    }
+
+    const runtimeParams = {
+      rootDirectoryUrl,
+      webServer,
+
+      coverageEnabled,
+      coverageConfig,
+      coverageMethodForBrowsers,
+      coverageMethodForNodeJs,
+      isTestPlan: true,
+      teardown,
+    };
+
+    if (logMergeForCompletedExecutions && !process.stdout.isTTY) {
+      logMergeForCompletedExecutions = false;
+      logger.debug(
+        `Force logMergeForCompletedExecutions to false because process.stdout.isTTY is false`,
+      );
+    }
+    const debugLogsEnabled = logger.levels.debug;
+    const executionLogsEnabled = logger.levels.info;
+    const executionSpinner =
+      logRefresh &&
+      maxExecutionsInParallel === 1 &&
+      !debugLogsEnabled &&
+      executionLogsEnabled &&
+      process.stdout.isTTY &&
+      // if there is an error during execution npm will mess up the output
+      // (happens when npm runs several command in a workspace)
+      // so we enable spinner only when !process.exitCode (no error so far)
+      process.exitCode !== 1;
+
+    const startMs = Date.now();
+    let rawOutput = "";
+    let executionLog = createLog({ newLine: "" });
+    const counters = {
+      total: executionSteps.length,
+      aborted: 0,
+      timedout: 0,
+      failed: 0,
+      completed: 0,
+      done: 0,
+    };
+
+    const callWhenPreviousExecutionAreDone = createCallOrderer();
+
+    await executeInParallel({
+      multipleExecutionsOperation,
+      maxExecutionsInParallel,
+      cooldownBetweenExecutions,
+      executionSteps,
+      start: async (paramsFromStep) => {
+        const executionIndex = executionSteps.indexOf(paramsFromStep);
+        const { executionName, fileRelativeUrl, runtime } = paramsFromStep;
+        const runtimeType = runtime.type;
+        const runtimeName = runtime.name;
+        const runtimeVersion = runtime.version;
+        const executionParams = {
+          measurePerformance: false,
+          collectPerformance: false,
+          collectConsole: true,
+          allocatedMs: defaultMsAllocatedPerExecution,
+          ...paramsFromStep,
+          runtimeParams: {
+            fileRelativeUrl,
+            ...paramsFromStep.runtimeParams,
+          },
+        };
+        const beforeExecutionInfo = {
+          fileRelativeUrl,
+          runtimeType,
+          runtimeName,
+          runtimeVersion,
+          executionIndex,
+          executionParams,
+          startMs: Date.now(),
+          executionResult: {
+            status: "executing",
+          },
+          counters,
+          timeEllapsed: Date.now() - startMs,
+          memoryHeap: memoryUsage().heapUsed,
+        };
+        if (typeof executionParams.allocatedMs === "function") {
+          executionParams.allocatedMs =
+            executionParams.allocatedMs(beforeExecutionInfo);
+        }
+        let spinner;
+        if (executionSpinner) {
+          spinner = startSpinner({
+            log: executionLog,
+            render: () => {
+              return createExecutionLog(beforeExecutionInfo, {
+                logRuntime,
+                logEachDuration,
+                logTimeUsage,
+                logMemoryHeapUsage,
+              });
+            },
+          });
+        }
+        for (const beforeExecutionCallback of beforeExecutionCallbackSet) {
+          beforeExecutionCallback(beforeExecutionInfo);
+        }
+        const fileUrl = `${rootDirectoryUrl}${fileRelativeUrl}`;
+        let executionResult;
+        if (existsSync(new URL(fileUrl))) {
+          executionResult = await run({
+            signal: multipleExecutionsOperation.signal,
+            logger,
+            allocatedMs: executionParams.allocatedMs,
+            keepRunning,
+            mirrorConsole: false, // might be executed in parallel: log would be a mess to read
+            collectConsole: executionParams.collectConsole,
+            coverageEnabled,
+            coverageTempDirectoryUrl,
+            runtime: executionParams.runtime,
+            runtimeParams: {
+              ...runtimeParams,
+              ...executionParams.runtimeParams,
+            },
+          });
+        } else {
+          executionResult = {
+            status: "failed",
+            errors: [
+              new Error(
+                `No file at ${fileRelativeUrl} for execution "${executionName}"`,
+              ),
+            ],
+          };
+        }
+        const fileReport = report[fileRelativeUrl];
+        if (fileReport) {
+          fileReport[executionName] = executionResult;
+        } else {
+          report[fileRelativeUrl] = {
+            [executionName]: executionResult,
+          };
+        }
+
+        const afterExecutionInfo = {
+          ...beforeExecutionInfo,
+          runtimeVersion: runtime.version,
+          endMs: Date.now(),
+          executionResult,
+        };
+
+        if (gcBetweenExecutions) {
+          global.gc();
+        }
+
+        const timeEllapsed = Date.now() - startMs;
+        const memoryHeap = memoryUsage().heapUsed;
+        callWhenPreviousExecutionAreDone(executionIndex, () => {
+          counters.done++;
+          if (executionResult.status === "aborted") {
+            counters.aborted++;
+          } else if (executionResult.status === "timedout") {
+            counters.timedout++;
+          } else if (executionResult.status === "failed") {
+            counters.failed++;
+          } else if (executionResult.status === "completed") {
+            counters.completed++;
+          }
+
+          if (executionLogsEnabled) {
+            // replace spinner with this execution result
+            if (spinner) {
+              spinner.stop();
+              spinner = null;
+            }
+
+            const log = createExecutionLog(afterExecutionInfo, {
+              logShortForCompletedExecutions,
+              logRuntime,
+              logEachDuration,
+              ...(logTimeUsage ? { timeEllapsed } : {}),
+              ...(logMemoryHeapUsage ? { memoryHeap } : {}),
+            });
+
+            executionLog.write(log);
+            rawOutput += stripAnsi(log);
+            const canOverwriteLog = canOverwriteLogGetter({
+              logMergeForCompletedExecutions,
+              executionResult,
+            });
+            if (canOverwriteLog) {
+              // nothing to do, we reuse the current executionLog object
+            } else {
+              executionLog.destroy();
+              executionLog = createLog({ newLine: "" });
+            }
+            const isLastExecutionLog =
+              executionIndex === executionSteps.length - 1;
+            if (isLastExecutionLog && logger.levels.info) {
+              executionLog.write("\n");
+            }
+          }
+        });
+        for (const afterExecutionCallback of afterExecutionCallbackSet) {
+          afterExecutionCallback(afterExecutionInfo);
+        }
+        const cancelRemaining =
+          failFast &&
+          executionResult.status !== "completed" &&
+          counters.done < counters.total;
+        if (cancelRemaining) {
+          logger.info(`"failFast" enabled -> cancel remaining executions`);
+          failFastAbortController.abort();
+        }
+      },
+    });
+    if (!keepRunning) {
+      logger.debug("trigger test plan teardown");
+      await teardown.trigger();
+    }
+
+    counters.cancelled = counters.total - counters.done;
+    const summary = {
+      counters,
+      // when execution is aborted, the remaining executions are "cancelled"
+      duration: Date.now() - startMs,
+    };
+    if (logSummary) {
+      const summaryLog = formatSummaryLog(summary);
+      rawOutput += stripAnsi(summaryLog);
+      logger.info(summaryLog);
+    }
+    if (summary.counters.total !== summary.counters.completed) {
+      const logFileUrl = new URL(logFileRelativeUrl, rootDirectoryUrl).href;
+      writeFileSync(logFileUrl, rawOutput);
+      logger.info(`-> ${urlToFileSystemPath(logFileUrl)}`);
+    }
+    returnValue.aborted = multipleExecutionsOperation.signal.aborted;
+    returnValue.summary = summary;
+    returnValue.report = report;
+    for (const callback of callbacks) {
+      await callback();
+    }
+  } finally {
+    await multipleExecutionsOperation.end();
+  }
 
   const hasFailed =
-    result.planSummary.counters.total !== result.planSummary.counters.completed;
+    returnValue.summary.counters.total !==
+    returnValue.summary.counters.completed;
   if (updateProcessExitCode && hasFailed) {
     process.exitCode = 1;
   }
-  const planCoverage = result.planCoverage;
+  const coverage = returnValue.coverage;
   // planCoverage can be null when execution is aborted
-  if (planCoverage) {
+  if (coverage) {
     const promises = [];
     // keep this one first because it does ensureEmptyDirectory
     // and in case coverage json file gets written in the same directory
@@ -477,7 +760,7 @@ To fix this warning:
         `-> ${urlToFileSystemPath(htmlCoverageDirectoryIndexFileUrl)}`,
       );
       promises.push(
-        generateCoverageHtmlDirectory(planCoverage, {
+        generateCoverageHtmlDirectory(coverage, {
           rootDirectoryUrl,
           coverageHtmlDirectoryRelativeUrl: urlToRelativeUrl(
             coverageReportHtmlDirectoryUrl,
@@ -491,7 +774,7 @@ To fix this warning:
     if (coverageEnabled && coverageReportJson) {
       promises.push(
         generateCoverageJsonFile({
-          coverage: result.planCoverage,
+          coverage,
           coverageJsonFileUrl: coverageReportJsonFileUrl,
           logger,
         }),
@@ -499,7 +782,7 @@ To fix this warning:
     }
     if (coverageEnabled && coverageReportTextLog) {
       promises.push(
-        generateCoverageTextLog(result.planCoverage, {
+        generateCoverageTextLog(coverage, {
           coverageReportSkipEmpty,
           coverageReportSkipFull,
         }),
@@ -508,12 +791,81 @@ To fix this warning:
     await Promise.all(promises);
   }
 
-  const returnValue = {
-    testPlanAborted: result.aborted,
-    testPlanSummary: result.planSummary,
-    testPlanReport: result.planReport,
-    testPlanCoverage: planCoverage,
-  };
-  await afterAllExecutionCallback(returnValue);
+  for (const afterAllExecutionCallback of afterAllExecutionCallbackSet) {
+    await afterAllExecutionCallback(returnValue);
+  }
   return returnValue;
 };
+
+const canOverwriteLogGetter = ({
+  logMergeForCompletedExecutions,
+  executionResult,
+}) => {
+  if (!logMergeForCompletedExecutions) {
+    return false;
+  }
+  if (executionResult.status === "aborted") {
+    return true;
+  }
+  if (executionResult.status !== "completed") {
+    return false;
+  }
+  const { consoleCalls = [] } = executionResult;
+  if (consoleCalls.length > 0) {
+    return false;
+  }
+  return true;
+};
+
+const executeInParallel = async ({
+  multipleExecutionsOperation,
+  maxExecutionsInParallel,
+  cooldownBetweenExecutions,
+  executionSteps,
+  start,
+}) => {
+  const executionResults = [];
+  let progressionIndex = 0;
+  let remainingExecutionCount = executionSteps.length;
+
+  const nextChunk = async () => {
+    if (multipleExecutionsOperation.signal.aborted) {
+      return;
+    }
+    const outputPromiseArray = [];
+    let previousExecPromise = Promise.resolve();
+    while (
+      remainingExecutionCount > 0 &&
+      outputPromiseArray.length < maxExecutionsInParallel
+    ) {
+      remainingExecutionCount--;
+      const outputPromise = executeOne(progressionIndex, previousExecPromise);
+      previousExecPromise = outputPromise;
+      progressionIndex++;
+      outputPromiseArray.push(outputPromise);
+    }
+    if (outputPromiseArray.length) {
+      await Promise.all(outputPromiseArray);
+      if (remainingExecutionCount > 0) {
+        await nextChunk();
+      }
+    }
+  };
+
+  const executeOne = async (index, previousExecPromise) => {
+    const input = executionSteps[index];
+    const output = await start(input, previousExecPromise);
+    if (!multipleExecutionsOperation.signal.aborted) {
+      executionResults[index] = output;
+    }
+    if (cooldownBetweenExecutions) {
+      await new Promise((resolve) =>
+        setTimeout(resolve, cooldownBetweenExecutions),
+      );
+    }
+  };
+
+  await nextChunk();
+
+  return executionResults;
+};

package/src/helpers/call_orderer.js

@@ -0,0 +1,39 @@
+export const createCallOrderer = () => {
+  const queue = [];
+  const callWhenPreviousExecutionAreDone = (executionIndex, callback) => {
+    if (queue[executionIndex]) {
+      throw new Error(`${executionIndex} already used`);
+    }
+
+    let allBeforeAreDone = true;
+    if (executionIndex > 0) {
+      let beforeIndex = executionIndex - 1;
+      do {
+        const value = queue[beforeIndex];
+        if (!value) {
+          allBeforeAreDone = false;
+          break;
+        }
+      } while (beforeIndex--);
+    }
+    if (!allBeforeAreDone) {
+      queue[executionIndex] = callback;
+      return;
+    }
+    queue[executionIndex] = true;
+    callback();
+    let afterIndex = executionIndex + 1;
+    while (afterIndex < queue.length) {
+      const value = queue[afterIndex];
+      if (value === undefined) {
+        break;
+      }
+      if (typeof value === "function") {
+        queue[afterIndex] = true;
+        value();
+      }
+      afterIndex++;
+    }
+  };
+  return callWhenPreviousExecutionAreDone;
+};