@jsenv/test 1.0.0

package/src/coverage/v8_coverage_node_directory.js

@@ -0,0 +1,85 @@
+import { readFileSync, readdirSync } from "node:fs"
+import { assertAndNormalizeDirectoryUrl } from "@jsenv/filesystem"
+import { createDetailedMessage } from "@jsenv/log"
+import { Abort } from "@jsenv/abort"
+
+export const readNodeV8CoverageDirectory = async ({
+  logger,
+  signal,
+  onV8Coverage,
+  maxMsWaitingForNodeToWriteCoverageFile = 2000,
+}) => {
+  const NODE_V8_COVERAGE = process.env.NODE_V8_COVERAGE
+  const operation = Abort.startOperation()
+  operation.addAbortSignal(signal)
+
+  let timeSpentTrying = 0
+  const tryReadDirectory = async () => {
+    const dirContent = readdirSync(NODE_V8_COVERAGE)
+    if (dirContent.length > 0) {
+      return dirContent
+    }
+    if (timeSpentTrying < maxMsWaitingForNodeToWriteCoverageFile) {
+      await new Promise((resolve) => setTimeout(resolve, 200))
+      timeSpentTrying += 200
+      logger.debug("retry to read coverage directory")
+      return tryReadDirectory()
+    }
+    logger.warn(`v8 coverage directory is empty at ${NODE_V8_COVERAGE}`)
+    return dirContent
+  }
+
+  try {
+    operation.throwIfAborted()
+    const dirContent = await tryReadDirectory()
+
+    const coverageDirectoryUrl = assertAndNormalizeDirectoryUrl(
+      NODE_V8_COVERAGE,
+      "NODE_V8_COVERAGE",
+    )
+
+    await dirContent.reduce(async (previous, dirEntry) => {
+      operation.throwIfAborted()
+      await previous
+
+      const dirEntryUrl = new URL(dirEntry, coverageDirectoryUrl)
+      const tryReadJsonFile = async () => {
+        const fileContent = String(readFileSync(dirEntryUrl))
+        if (fileContent === "") {
+          if (timeSpentTrying < maxMsWaitingForNodeToWriteCoverageFile) {
+            await new Promise((resolve) => setTimeout(resolve, 200))
+            timeSpentTrying += 200
+            return tryReadJsonFile()
+          }
+          console.warn(`Coverage JSON file is empty at ${dirEntryUrl}`)
+          return null
+        }
+
+        try {
+          const fileAsJson = JSON.parse(fileContent)
+          return fileAsJson
+        } catch (e) {
+          if (timeSpentTrying < maxMsWaitingForNodeToWriteCoverageFile) {
+            await new Promise((resolve) => setTimeout(resolve, 200))
+            timeSpentTrying += 200
+            return tryReadJsonFile()
+          }
+          console.warn(
+            createDetailedMessage(`Error while reading coverage file`, {
+              "error stack": e.stack,
+              "file": dirEntryUrl,
+            }),
+          )
+          return null
+        }
+      }
+
+      const fileContent = await tryReadJsonFile()
+      if (fileContent) {
+        await onV8Coverage(fileContent)
+      }
+    }, Promise.resolve())
+  } finally {
+    await operation.end()
+  }
+}

package/src/coverage/v8_coverage_to_istanbul.js

@@ -0,0 +1,99 @@
+import { urlToFileSystemPath } from "@jsenv/urls"
+import { Abort } from "@jsenv/abort"
+import { requireFromJsenv } from "@jsenv/core/src/helpers/require_from_jsenv.js"
+
+import { composeTwoFileByFileIstanbulCoverages } from "./istanbul_coverage_composition.js"
+
+export const v8CoverageToIstanbul = async (v8Coverage, { signal }) => {
+  const operation = Abort.startOperation()
+  operation.addAbortSignal(signal)
+
+  try {
+    const v8ToIstanbul = requireFromJsenv("v8-to-istanbul")
+    const sourcemapCache = v8Coverage["source-map-cache"]
+    let istanbulCoverageComposed = null
+
+    await v8Coverage.result.reduce(async (previous, fileV8Coverage) => {
+      operation.throwIfAborted()
+      await previous
+
+      const { source } = fileV8Coverage
+      let sources
+      // when v8 coverage comes from playwright (chromium) v8Coverage.source is set
+      if (typeof source === "string") {
+        sources = { source }
+      }
+      // when v8 coverage comes from Node.js, the source can be read from sourcemapCache
+      else if (sourcemapCache) {
+        sources = sourcesFromSourceMapCache(fileV8Coverage.url, sourcemapCache)
+      }
+      const path = urlToFileSystemPath(fileV8Coverage.url)
+
+      const converter = v8ToIstanbul(
+        path,
+        // wrapperLength is undefined we don't need it
+        // https://github.com/istanbuljs/v8-to-istanbul/blob/2b54bc97c5edf8a37b39a171ec29134ba9bfd532/lib/v8-to-istanbul.js#L27
+        undefined,
+        sources,
+      )
+      await converter.load()
+
+      converter.applyCoverage(fileV8Coverage.functions)
+      const istanbulCoverage = converter.toIstanbul()
+
+      istanbulCoverageComposed = istanbulCoverageComposed
+        ? composeTwoFileByFileIstanbulCoverages(
+            istanbulCoverageComposed,
+            istanbulCoverage,
+          )
+        : istanbulCoverage
+    }, Promise.resolve())
+
+    if (!istanbulCoverageComposed) {
+      return {}
+    }
+    istanbulCoverageComposed = markAsConvertedFromV8(istanbulCoverageComposed)
+    return istanbulCoverageComposed
+  } finally {
+    await operation.end()
+  }
+}
+
+const markAsConvertedFromV8 = (fileByFileCoverage) => {
+  const fileByFileMarked = {}
+  Object.keys(fileByFileCoverage).forEach((key) => {
+    const fileCoverage = fileByFileCoverage[key]
+    fileByFileMarked[key] = {
+      ...fileCoverage,
+      fromV8: true,
+    }
+  })
+  return fileByFileMarked
+}
+
+const sourcesFromSourceMapCache = (url, sourceMapCache) => {
+  const sourceMapAndLineLengths = sourceMapCache[url]
+  if (!sourceMapAndLineLengths) {
+    return {}
+  }
+
+  const { data, lineLengths } = sourceMapAndLineLengths
+  // See: https://github.com/nodejs/node/pull/34305
+  if (!data) {
+    return undefined
+  }
+
+  const sources = {
+    sourcemap: data,
+    ...(lineLengths ? { source: sourcesFromLineLengths(lineLengths) } : {}),
+  }
+  return sources
+}
+
+const sourcesFromLineLengths = (lineLengths) => {
+  let source = ""
+  lineLengths.forEach((length) => {
+    source += `${"".padEnd(length, ".")}\n`
+  })
+  return source
+}

package/src/execution/execute.js

@@ -0,0 +1,111 @@
+/*
+ * Export a function capable to execute a file on a runtime (browser or node) and return how it goes.
+ *
+ * - can be useful to execute a file in a browser/node.js programmatically
+ * - not documented
+ * - the most importants parts:
+ *   - fileRelativeUrl: the file to execute inside rootDirectoryUrl
+ *   - runtime: an object with a "run" method.
+ *   The run method will start a browser/node process and execute file in it
+ * - Most of the logic lives in "./run.js" used by executeTestPlan to run tests
+ */
+
+import { Abort, raceProcessTeardownEvents } from "@jsenv/abort"
+import { assertAndNormalizeDirectoryUrl } from "@jsenv/filesystem"
+import { createLogger } from "@jsenv/log"
+
+import { assertAndNormalizeWebServer } from "./web_server_param.js"
+import { run } from "./run.js"
+
+export const execute = async ({
+  signal = new AbortController().signal,
+  handleSIGINT = true,
+  logLevel,
+  rootDirectoryUrl,
+  webServer,
+  importMap,
+
+  fileRelativeUrl,
+  allocatedMs,
+  mirrorConsole = true,
+  keepRunning = false,
+
+  collectConsole,
+  collectCoverage,
+  coverageTempDirectoryUrl,
+  collectPerformance = false,
+  runtime,
+  runtimeParams,
+
+  ignoreError = false,
+}) => {
+  const logger = createLogger({ logLevel })
+  rootDirectoryUrl = assertAndNormalizeDirectoryUrl(
+    rootDirectoryUrl,
+    "rootDirectoryUrl",
+  )
+  const executeOperation = Abort.startOperation()
+  executeOperation.addAbortSignal(signal)
+  if (handleSIGINT) {
+    executeOperation.addAbortSource((abort) => {
+      return raceProcessTeardownEvents(
+        {
+          SIGINT: true,
+        },
+        abort,
+      )
+    })
+  }
+
+  if (runtime.type === "browser") {
+    await assertAndNormalizeWebServer(webServer)
+  }
+
+  let resultTransformer = (result) => result
+  runtimeParams = {
+    rootDirectoryUrl,
+    webServer,
+    fileRelativeUrl,
+    importMap,
+    ...runtimeParams,
+  }
+
+  let result = await run({
+    signal: executeOperation.signal,
+    logger,
+    allocatedMs,
+    keepRunning,
+    mirrorConsole,
+    collectConsole,
+    collectCoverage,
+    coverageTempDirectoryUrl,
+    collectPerformance,
+    runtime,
+    runtimeParams,
+  })
+  result = resultTransformer(result)
+
+  try {
+    if (result.status === "failed") {
+      if (ignoreError) {
+        return result
+      }
+      /*
+  Warning: when node launched with --unhandled-rejections=strict, despites
+  this promise being rejected by throw result.error node will completely ignore it.
+
+  The error can be logged by doing
+  ```js
+  process.setUncaughtExceptionCaptureCallback((error) => {
+    console.error(error.stack)
+  })
+  ```
+  But it feels like a hack.
+  */
+      throw result.errors[result.errors.length - 1]
+    }
+    return result
+  } finally {
+    await executeOperation.end()
+  }
+}

package/src/execution/execute_steps.js

@@ -0,0 +1,423 @@
+import { existsSync } from "node:fs"
+import { memoryUsage } from "node:process"
+import { takeCoverage } from "node:v8"
+import stripAnsi from "strip-ansi"
+import { urlToFileSystemPath } from "@jsenv/urls"
+import { createLog, startSpinner } from "@jsenv/log"
+import { Abort, raceProcessTeardownEvents } from "@jsenv/abort"
+import { ensureEmptyDirectory, writeFileSync } from "@jsenv/filesystem"
+
+import { reportToCoverage } from "../coverage/report_to_coverage.js"
+import { run } from "./run.js"
+import { ensureGlobalGc } from "./gc.js"
+import { createExecutionLog, createSummaryLog } from "./logs_file_execution.js"
+
+export const executeSteps = async (
+  executionSteps,
+  {
+    signal,
+    handleSIGINT,
+    logger,
+    logRefresh,
+    logRuntime,
+    logEachDuration,
+    logSummary,
+    logTimeUsage,
+    logMemoryHeapUsage,
+    logFileRelativeUrl,
+    completedExecutionLogMerging,
+    completedExecutionLogAbbreviation,
+    rootDirectoryUrl,
+    webServer,
+
+    keepRunning,
+    defaultMsAllocatedPerExecution,
+    maxExecutionsInParallel,
+    failFast,
+    gcBetweenExecutions,
+    cooldownBetweenExecutions,
+
+    coverageEnabled,
+    coverageConfig,
+    coverageIncludeMissing,
+    coverageMethodForBrowsers,
+    coverageMethodForNodeJs,
+    coverageV8ConflictWarning,
+    coverageTempDirectoryUrl,
+
+    beforeExecutionCallback = () => {},
+    afterExecutionCallback = () => {},
+  } = {},
+) => {
+  const executePlanReturnValue = {}
+  const report = {}
+  const callbacks = []
+  const stopAfterAllSignal = { notify: () => {} }
+
+  const multipleExecutionsOperation = Abort.startOperation()
+  multipleExecutionsOperation.addAbortSignal(signal)
+  if (handleSIGINT) {
+    multipleExecutionsOperation.addAbortSource((abort) => {
+      return raceProcessTeardownEvents(
+        {
+          SIGINT: true,
+        },
+        () => {
+          logger.debug(`SIGINT abort`)
+          abort()
+        },
+      )
+    })
+  }
+  const failFastAbortController = new AbortController()
+  if (failFast) {
+    multipleExecutionsOperation.addAbortSignal(failFastAbortController.signal)
+  }
+
+  try {
+    if (gcBetweenExecutions) {
+      ensureGlobalGc()
+    }
+
+    if (coverageEnabled) {
+      // when runned multiple times, we don't want to keep previous files in this directory
+      await ensureEmptyDirectory(coverageTempDirectoryUrl)
+      callbacks.push(async () => {
+        if (multipleExecutionsOperation.signal.aborted) {
+          // don't try to do the coverage stuff
+          return
+        }
+        try {
+          if (coverageMethodForNodeJs === "NODE_V8_COVERAGE") {
+            takeCoverage()
+            // conceptually we don't need coverage anymore so it would be
+            // good to call v8.stopCoverage()
+            // but it logs a strange message about "result is not an object"
+          }
+          const planCoverage = await reportToCoverage(report, {
+            signal: multipleExecutionsOperation.signal,
+            logger,
+            rootDirectoryUrl,
+            coverageConfig,
+            coverageIncludeMissing,
+            coverageMethodForBrowsers,
+            coverageV8ConflictWarning,
+          })
+          executePlanReturnValue.planCoverage = planCoverage
+        } catch (e) {
+          if (Abort.isAbortError(e)) {
+            return
+          }
+          throw e
+        }
+      })
+    }
+
+    const runtimeParams = {
+      rootDirectoryUrl,
+      webServer,
+
+      coverageEnabled,
+      coverageConfig,
+      coverageMethodForBrowsers,
+      coverageMethodForNodeJs,
+      stopAfterAllSignal,
+    }
+
+    if (completedExecutionLogMerging && !process.stdout.isTTY) {
+      completedExecutionLogMerging = false
+      logger.debug(
+        `Force completedExecutionLogMerging to false because process.stdout.isTTY is false`,
+      )
+    }
+    const debugLogsEnabled = logger.levels.debug
+    const executionLogsEnabled = logger.levels.info
+    const executionSpinner =
+      logRefresh &&
+      !debugLogsEnabled &&
+      executionLogsEnabled &&
+      process.stdout.isTTY &&
+      // if there is an error during execution npm will mess up the output
+      // (happens when npm runs several command in a workspace)
+      // so we enable spinner only when !process.exitCode (no error so far)
+      process.exitCode !== 1
+
+    const startMs = Date.now()
+    let rawOutput = ""
+    let executionLog = createLog({ newLine: "" })
+    const counters = {
+      total: executionSteps.length,
+      aborted: 0,
+      timedout: 0,
+      failed: 0,
+      completed: 0,
+      done: 0,
+    }
+    await executeInParallel({
+      multipleExecutionsOperation,
+      maxExecutionsInParallel,
+      cooldownBetweenExecutions,
+      executionSteps,
+      start: async (paramsFromStep) => {
+        const executionIndex = executionSteps.indexOf(paramsFromStep)
+        const { executionName, fileRelativeUrl, runtime } = paramsFromStep
+        const runtimeType = runtime.type
+        const runtimeName = runtime.name
+        const runtimeVersion = runtime.version
+        const executionParams = {
+          measurePerformance: false,
+          collectPerformance: false,
+          collectConsole: true,
+          allocatedMs: defaultMsAllocatedPerExecution,
+          ...paramsFromStep,
+          runtimeParams: {
+            fileRelativeUrl,
+            ...paramsFromStep.runtimeParams,
+          },
+        }
+        const beforeExecutionInfo = {
+          fileRelativeUrl,
+          runtimeType,
+          runtimeName,
+          runtimeVersion,
+          executionIndex,
+          executionParams,
+          startMs: Date.now(),
+          executionResult: {
+            status: "executing",
+          },
+        }
+        let spinner
+        if (executionSpinner) {
+          spinner = startSpinner({
+            log: executionLog,
+            render: () => {
+              return createExecutionLog(beforeExecutionInfo, {
+                counters,
+                logRuntime,
+                logEachDuration,
+                ...(logTimeUsage
+                  ? {
+                      timeEllapsed: Date.now() - startMs,
+                    }
+                  : {}),
+                ...(logMemoryHeapUsage
+                  ? { memoryHeap: memoryUsage().heapUsed }
+                  : {}),
+              })
+            },
+          })
+        }
+        beforeExecutionCallback(beforeExecutionInfo)
+
+        const fileUrl = `${rootDirectoryUrl}${fileRelativeUrl}`
+        let executionResult
+        if (existsSync(new URL(fileUrl))) {
+          executionResult = await run({
+            signal: multipleExecutionsOperation.signal,
+            logger,
+            allocatedMs:
+              typeof executionParams.allocatedMs === "function"
+                ? executionParams.allocatedMs(beforeExecutionInfo)
+                : executionParams.allocatedMs,
+            keepRunning,
+            mirrorConsole: false, // file are executed in parallel, log would be a mess to read
+            collectConsole: executionParams.collectConsole,
+            coverageEnabled,
+            coverageTempDirectoryUrl,
+            runtime: executionParams.runtime,
+            runtimeParams: {
+              ...runtimeParams,
+              ...executionParams.runtimeParams,
+            },
+          })
+        } else {
+          executionResult = {
+            status: "failed",
+            errors: [
+              new Error(
+                `No file at ${fileRelativeUrl} for execution "${executionName}"`,
+              ),
+            ],
+          }
+        }
+        counters.done++
+        const fileReport = report[fileRelativeUrl]
+        if (fileReport) {
+          fileReport[executionName] = executionResult
+        } else {
+          report[fileRelativeUrl] = {
+            [executionName]: executionResult,
+          }
+        }
+
+        const afterExecutionInfo = {
+          ...beforeExecutionInfo,
+          runtimeVersion: runtime.version,
+          endMs: Date.now(),
+          executionResult,
+        }
+        afterExecutionCallback(afterExecutionInfo)
+
+        if (executionResult.status === "aborted") {
+          counters.aborted++
+        } else if (executionResult.status === "timedout") {
+          counters.timedout++
+        } else if (executionResult.status === "failed") {
+          counters.failed++
+        } else if (executionResult.status === "completed") {
+          counters.completed++
+        }
+        if (gcBetweenExecutions) {
+          global.gc()
+        }
+        if (executionLogsEnabled) {
+          const log = createExecutionLog(afterExecutionInfo, {
+            completedExecutionLogAbbreviation,
+            counters,
+            logRuntime,
+            logEachDuration,
+            ...(logTimeUsage
+              ? {
+                  timeEllapsed: Date.now() - startMs,
+                }
+              : {}),
+            ...(logMemoryHeapUsage
+              ? { memoryHeap: memoryUsage().heapUsed }
+              : {}),
+          })
+          // replace spinner with this execution result
+          if (spinner) spinner.stop()
+          executionLog.write(log)
+          rawOutput += stripAnsi(log)
+
+          const canOverwriteLog = canOverwriteLogGetter({
+            completedExecutionLogMerging,
+            executionResult,
+          })
+          if (canOverwriteLog) {
+            // nothing to do, we reuse the current executionLog object
+          } else {
+            executionLog.destroy()
+            executionLog = createLog({ newLine: "" })
+          }
+        }
+        const isLastExecutionLog = executionIndex === executionSteps.length - 1
+        const cancelRemaining =
+          failFast &&
+          executionResult.status !== "completed" &&
+          counters.done < counters.total
+        if (isLastExecutionLog && logger.levels.info) {
+          executionLog.write("\n")
+        }
+
+        if (cancelRemaining) {
+          logger.info(`"failFast" enabled -> cancel remaining executions`)
+          failFastAbortController.abort()
+        }
+      },
+    })
+    if (!keepRunning) {
+      logger.debug("stopAfterAllSignal.notify()")
+      await stopAfterAllSignal.notify()
+    }
+
+    counters.cancelled = counters.total - counters.done
+    const summary = {
+      counters,
+      // when execution is aborted, the remaining executions are "cancelled"
+      duration: Date.now() - startMs,
+    }
+    if (logSummary) {
+      const summaryLog = createSummaryLog(summary)
+      rawOutput += stripAnsi(summaryLog)
+      logger.info(summaryLog)
+    }
+    if (summary.counters.total !== summary.counters.completed) {
+      const logFileUrl = new URL(logFileRelativeUrl, rootDirectoryUrl).href
+      writeFileSync(logFileUrl, rawOutput)
+      logger.info(`-> ${urlToFileSystemPath(logFileUrl)}`)
+    }
+    executePlanReturnValue.aborted = multipleExecutionsOperation.signal.aborted
+    executePlanReturnValue.planSummary = summary
+    executePlanReturnValue.planReport = report
+    await callbacks.reduce(async (previous, callback) => {
+      await previous
+      await callback()
+    }, Promise.resolve())
+    return executePlanReturnValue
+  } finally {
+    await multipleExecutionsOperation.end()
+  }
+}
+
+const canOverwriteLogGetter = ({
+  completedExecutionLogMerging,
+  executionResult,
+}) => {
+  if (!completedExecutionLogMerging) {
+    return false
+  }
+  if (executionResult.status === "aborted") {
+    return true
+  }
+  if (executionResult.status !== "completed") {
+    return false
+  }
+  const { consoleCalls = [] } = executionResult
+  if (consoleCalls.length > 0) {
+    return false
+  }
+  return true
+}
+
+const executeInParallel = async ({
+  multipleExecutionsOperation,
+  maxExecutionsInParallel,
+  cooldownBetweenExecutions,
+  executionSteps,
+  start,
+}) => {
+  const executionResults = []
+  let progressionIndex = 0
+  let remainingExecutionCount = executionSteps.length
+
+  const nextChunk = async () => {
+    if (multipleExecutionsOperation.signal.aborted) {
+      return
+    }
+    const outputPromiseArray = []
+    while (
+      remainingExecutionCount > 0 &&
+      outputPromiseArray.length < maxExecutionsInParallel
+    ) {
+      remainingExecutionCount--
+      const outputPromise = executeOne(progressionIndex)
+      progressionIndex++
+      outputPromiseArray.push(outputPromise)
+    }
+    if (outputPromiseArray.length) {
+      await Promise.all(outputPromiseArray)
+      if (remainingExecutionCount > 0) {
+        await nextChunk()
+      }
+    }
+  }
+
+  const executeOne = async (index) => {
+    const input = executionSteps[index]
+    const output = await start(input)
+    if (!multipleExecutionsOperation.signal.aborted) {
+      executionResults[index] = output
+    }
+    if (cooldownBetweenExecutions) {
+      await new Promise((resolve) =>
+        setTimeout(resolve, cooldownBetweenExecutions),
+      )
+    }
+  }
+
+  await nextChunk()
+
+  return executionResults
+}