@jsenv/core 25.4.9 → 26.0.0

package/src/internal/executing/execute_plan.js

@@ -0,0 +1,604 @@
+import { existsSync } from "node:fs"
+import { memoryUsage } from "node:process"
+import wrapAnsi from "wrap-ansi"
+import stripAnsi from "strip-ansi"
+import cuid from "cuid"
+import { createDetailedMessage, loggerToLevels } from "@jsenv/logger"
+import { createLog, startSpinner } from "@jsenv/log"
+import {
+  Abort,
+  raceProcessTeardownEvents,
+  createCallbackListNotifiedOnce,
+} from "@jsenv/abort"
+import {
+  urlToFileSystemPath,
+  resolveUrl,
+  writeDirectory,
+  ensureEmptyDirectory,
+  normalizeStructuredMetaMap,
+  urlToMeta,
+  writeFile,
+} from "@jsenv/filesystem"
+
+import { startCompileServer } from "@jsenv/core/src/internal/compile_server/compile_server.js"
+import { babelPluginInstrument } from "@jsenv/core/src/internal/coverage/babel_plugin_instrument.js"
+import { reportToCoverage } from "@jsenv/core/src/internal/coverage/report_to_coverage.js"
+
+import { ensureGlobalGc } from "./gc.js"
+import { launchAndExecute } from "./launch_and_execute.js"
+import { generateExecutionSteps } from "./execution_steps.js"
+import {
+  formatExecuting,
+  formatExecutionResult,
+} from "./logs_file_execution.js"
+import { createSummaryLog } from "./logs_plan_summary.js"
+
+export const executePlan = async (
+  plan,
+  {
+    signal,
+    handleSIGINT,
+
+    logger,
+    compileServerLogLevel,
+    launchAndExecuteLogLevel,
+
+    projectDirectoryUrl,
+    jsenvDirectoryRelativeUrl,
+    jsenvDirectoryClean,
+
+    importResolutionMethod,
+    importDefaultExtension,
+
+    logSummary,
+    logMemoryHeapUsage,
+    logFileRelativeUrl,
+    completedExecutionLogMerging,
+    completedExecutionLogAbbreviation,
+
+    defaultMsAllocatedPerExecution,
+    maxExecutionsInParallel,
+    failFast,
+    gcBetweenExecutions,
+    stopAfterExecute,
+    cooldownBetweenExecutions,
+
+    coverage,
+    coverageConfig,
+    coverageIncludeMissing,
+    coverageForceIstanbul,
+    coverageV8ConflictWarning,
+    coverageTempDirectoryRelativeUrl,
+
+    protocol,
+    privateKey,
+    certificate,
+    ip,
+    port,
+    compileServerCanReadFromFilesystem,
+    compileServerCanWriteOnFilesystem,
+    babelPluginMap,
+    babelConfigFileUrl,
+    preservedUrls,
+    workers,
+    serviceWorkers,
+    importMapInWebWorkers,
+    customCompilers,
+
+    beforeExecutionCallback = () => {},
+    afterExecutionCallback = () => {},
+  } = {},
+) => {
+  if (coverage) {
+    babelPluginMap = {
+      ...babelPluginMap,
+      "transform-instrument": [
+        babelPluginInstrument,
+        { projectDirectoryUrl, coverageConfig },
+      ],
+    }
+  }
+  const runtimes = {}
+  Object.keys(plan).forEach((filePattern) => {
+    const filePlan = plan[filePattern]
+    Object.keys(filePlan).forEach((executionName) => {
+      const executionConfig = filePlan[executionName]
+      const { runtime } = executionConfig
+      if (runtime) {
+        runtimes[runtime.name] = runtime.version
+      }
+    })
+  })
+  logger.debug(
+    createDetailedMessage(`Prepare executing plan`, {
+      runtimes: JSON.stringify(runtimes, null, "  "),
+    }),
+  )
+  const multipleExecutionsOperation = Abort.startOperation()
+  multipleExecutionsOperation.addAbortSignal(signal)
+  if (handleSIGINT) {
+    multipleExecutionsOperation.addAbortSource((abort) => {
+      return raceProcessTeardownEvents(
+        {
+          SIGINT: true,
+        },
+        () => {
+          logger.debug(`SIGINT abort`)
+          abort()
+        },
+      )
+    })
+  }
+  const failFastAbortController = new AbortController()
+  if (failFast) {
+    multipleExecutionsOperation.addAbortSignal(failFastAbortController.signal)
+  }
+
+  try {
+    const compileServer = await startCompileServer({
+      signal: multipleExecutionsOperation.signal,
+      logLevel: compileServerLogLevel,
+
+      projectDirectoryUrl,
+      jsenvDirectoryRelativeUrl,
+      jsenvDirectoryClean,
+
+      importResolutionMethod,
+      importDefaultExtension,
+
+      protocol,
+      privateKey,
+      certificate,
+      ip,
+      port,
+      compileServerCanReadFromFilesystem,
+      compileServerCanWriteOnFilesystem,
+      keepProcessAlive: true, // to be sure it stays alive
+      babelPluginMap,
+      babelConfigFileUrl,
+      preservedUrls,
+      workers,
+      serviceWorkers,
+      importMapInWebWorkers,
+      customCompilers,
+    })
+    babelPluginMap = compileServer.babelPluginMap
+    multipleExecutionsOperation.addEndCallback(async () => {
+      await compileServer.stop()
+    })
+    logger.debug(`Generate executions`)
+    const executionSteps = await getExecutionAsSteps({
+      plan,
+      compileServer,
+      multipleExecutionsOperation,
+      projectDirectoryUrl,
+    })
+    logger.debug(`${executionSteps.length} executions planned`)
+
+    if (completedExecutionLogMerging && !process.stdout.isTTY) {
+      completedExecutionLogMerging = false
+      logger.debug(
+        `Force completedExecutionLogMerging to false because process.stdout.isTTY is false`,
+      )
+    }
+    const executionLogsEnabled = loggerToLevels(logger).info
+    const executionSpinner = executionLogsEnabled && process.stdout.isTTY
+
+    const startMs = Date.now()
+    const report = {}
+    const executionCount = executionSteps.length
+    let rawOutput = ""
+
+    let transformReturnValue = (value) => value
+
+    if (gcBetweenExecutions) {
+      ensureGlobalGc()
+    }
+
+    const coverageTempDirectoryUrl = resolveUrl(
+      coverageTempDirectoryRelativeUrl,
+      projectDirectoryUrl,
+    )
+    const structuredMetaMapForCover = normalizeStructuredMetaMap(
+      {
+        cover: coverageConfig,
+      },
+      projectDirectoryUrl,
+    )
+    const coverageIgnorePredicate = (url) => {
+      return !urlToMeta({
+        url: resolveUrl(url, projectDirectoryUrl),
+        structuredMetaMap: structuredMetaMapForCover,
+      }).cover
+    }
+
+    if (coverage) {
+      // in case runned multiple times, we don't want to keep writing lot of files in this directory
+      if (!process.env.NODE_V8_COVERAGE) {
+        await ensureEmptyDirectory(coverageTempDirectoryUrl)
+      }
+      if (runtimes.node) {
+        // v8 coverage is written in a directoy and auto propagate to subprocesses
+        // through process.env.NODE_V8_COVERAGE.
+        if (!coverageForceIstanbul && !process.env.NODE_V8_COVERAGE) {
+          const v8CoverageDirectory = resolveUrl(
+            `./node_v8/${cuid()}`,
+            coverageTempDirectoryUrl,
+          )
+          await writeDirectory(v8CoverageDirectory, { allowUseless: true })
+          process.env.NODE_V8_COVERAGE =
+            urlToFileSystemPath(v8CoverageDirectory)
+        }
+      }
+
+      transformReturnValue = async (value) => {
+        if (multipleExecutionsOperation.signal.aborted) {
+          // don't try to do the coverage stuff
+          return value
+        }
+
+        try {
+          value.coverage = await reportToCoverage(value.report, {
+            signal: multipleExecutionsOperation.signal,
+            logger,
+            projectDirectoryUrl,
+            babelPluginMap,
+            coverageConfig,
+            coverageIncludeMissing,
+            coverageForceIstanbul,
+            coverageIgnorePredicate,
+            coverageV8ConflictWarning,
+          })
+        } catch (e) {
+          if (Abort.isAbortError(e)) {
+            return value
+          }
+          throw e
+        }
+        return value
+      }
+    }
+
+    logger.info("")
+    let executionLog = createLog({ newLine: "" })
+    let abortedCount = 0
+    let timedoutCount = 0
+    let erroredCount = 0
+    let completedCount = 0
+    const stopAfterAllExecutionCallbackList = createCallbackListNotifiedOnce()
+
+    let executionDoneCount = 0
+    await executeInParallel({
+      multipleExecutionsOperation,
+      maxExecutionsInParallel,
+      cooldownBetweenExecutions,
+      executionSteps,
+      start: async (paramsFromStep) => {
+        const executionIndex = executionSteps.indexOf(paramsFromStep)
+        const { executionName, fileRelativeUrl, runtime } = paramsFromStep
+        const runtimeName = runtime.name
+        const runtimeVersion = runtime.version
+
+        const executionParams = {
+          // the params below can be overriden by executionDefaultParams
+          measurePerformance: false,
+          collectPerformance: false,
+          captureConsole: true,
+          stopAfterExecute,
+          stopAfterExecuteReason: "execution-done",
+          allocatedMs: defaultMsAllocatedPerExecution,
+          ...paramsFromStep,
+          runtime,
+          // mirrorConsole: false because file will be executed in parallel
+          // so log would be a mess to read
+          mirrorConsole: false,
+        }
+
+        const beforeExecutionInfo = {
+          fileRelativeUrl,
+          runtimeName,
+          runtimeVersion,
+          executionIndex,
+          executionParams,
+        }
+
+        let spinner
+        if (executionSpinner) {
+          spinner = startSpinner({
+            log: executionLog,
+            text: formatExecuting(beforeExecutionInfo, {
+              executionCount,
+              abortedCount,
+              timedoutCount,
+              erroredCount,
+              completedCount,
+              ...(logMemoryHeapUsage
+                ? { memoryHeap: memoryUsage().heapUsed }
+                : {}),
+            }),
+          })
+        }
+        beforeExecutionCallback(beforeExecutionInfo)
+
+        const filePath = urlToFileSystemPath(
+          `${projectDirectoryUrl}${fileRelativeUrl}`,
+        )
+        let executionResult
+        if (existsSync(filePath)) {
+          executionResult = await launchAndExecute({
+            signal: multipleExecutionsOperation.signal,
+            launchAndExecuteLogLevel,
+
+            ...executionParams,
+            collectCoverage: coverage,
+            coverageTempDirectoryUrl,
+            runtimeParams: {
+              projectDirectoryUrl,
+              compileServerOrigin: compileServer.origin,
+              compileServerId: compileServer.id,
+              jsenvDirectoryRelativeUrl:
+                compileServer.jsenvDirectoryRelativeUrl,
+
+              collectCoverage: coverage,
+              coverageIgnorePredicate,
+              coverageForceIstanbul,
+              stopAfterAllExecutionCallbackList,
+              ...executionParams.runtimeParams,
+            },
+            executeParams: {
+              fileRelativeUrl,
+              ...executionParams.executeParams,
+            },
+            coverageV8ConflictWarning,
+          })
+        } else {
+          executionResult = {
+            status: "errored",
+            error: new Error(
+              `No file at ${fileRelativeUrl} for execution "${executionName}"`,
+            ),
+          }
+        }
+        executionDoneCount++
+        if (fileRelativeUrl in report === false) {
+          report[fileRelativeUrl] = {}
+        }
+        report[fileRelativeUrl][executionName] = executionResult
+        const afterExecutionInfo = {
+          ...beforeExecutionInfo,
+          endMs: Date.now(),
+          executionResult,
+        }
+        afterExecutionCallback(afterExecutionInfo)
+
+        if (executionResult.status === "aborted") {
+          abortedCount++
+        } else if (executionResult.status === "timedout") {
+          timedoutCount++
+        } else if (executionResult.status === "errored") {
+          erroredCount++
+        } else if (executionResult.status === "completed") {
+          completedCount++
+        }
+        if (gcBetweenExecutions) {
+          global.gc()
+        }
+        if (executionLogsEnabled) {
+          let log = formatExecutionResult(afterExecutionInfo, {
+            completedExecutionLogAbbreviation,
+            executionCount,
+            abortedCount,
+            timedoutCount,
+            erroredCount,
+            completedCount,
+            ...(logMemoryHeapUsage
+              ? { memoryHeap: memoryUsage().heapUsed }
+              : {}),
+          })
+          log = `${log}
+  
+`
+          const { columns = 80 } = process.stdout
+          log = wrapAnsi(log, columns, {
+            trim: false,
+            hard: true,
+            wordWrap: false,
+          })
+
+          // replace spinner with this execution result
+          if (spinner) spinner.stop()
+          executionLog.write(log)
+          rawOutput += stripAnsi(log)
+
+          const canOverwriteLog = canOverwriteLogGetter({
+            completedExecutionLogMerging,
+            executionResult,
+          })
+          if (canOverwriteLog) {
+            // nothing to do, we reuse the current executionLog object
+          } else {
+            executionLog.destroy()
+            executionLog = createLog({ newLine: "" })
+          }
+        }
+        if (
+          failFast &&
+          executionResult.status !== "completed" &&
+          executionDoneCount < executionCount
+        ) {
+          logger.info(`"failFast" enabled -> cancel remaining executions`)
+          failFastAbortController.abort()
+        }
+      },
+    })
+
+    if (stopAfterExecute) {
+      stopAfterAllExecutionCallbackList.notify()
+    }
+
+    const summaryCounts = reportToSummary(report)
+    const summary = {
+      executionCount,
+      ...summaryCounts,
+      // when execution is aborted, the remaining executions are "cancelled"
+      cancelledCount: executionCount - executionDoneCount,
+      duration: Date.now() - startMs,
+    }
+    if (logSummary) {
+      const summaryLog = createSummaryLog(summary)
+      rawOutput += stripAnsi(summaryLog)
+      logger.info(summaryLog)
+    }
+    if (summary.executionCount !== summary.completedCount) {
+      const logFileUrl = new URL(logFileRelativeUrl, projectDirectoryUrl)
+      writeFile(logFileUrl, rawOutput)
+      logger.info(`-> ${urlToFileSystemPath(logFileUrl)}`)
+    }
+    const result = await transformReturnValue({
+      summary,
+      report,
+    })
+    return {
+      aborted: multipleExecutionsOperation.signal.aborted,
+      planSummary: result.summary,
+      planReport: result.report,
+      planCoverage: result.coverage,
+    }
+  } finally {
+    await multipleExecutionsOperation.end()
+  }
+}
+
+const getExecutionAsSteps = async ({
+  plan,
+  compileServer,
+  multipleExecutionsOperation,
+  projectDirectoryUrl,
+}) => {
+  try {
+    const executionSteps = await generateExecutionSteps(
+      {
+        ...plan,
+        [compileServer.jsenvDirectoryRelativeUrl]: null,
+      },
+      {
+        signal: multipleExecutionsOperation.signal,
+        projectDirectoryUrl,
+      },
+    )
+    return executionSteps
+  } catch (e) {
+    if (Abort.isAbortError(e)) {
+      return {
+        aborted: true,
+        planSummary: {},
+        planReport: {},
+        planCoverage: null,
+      }
+    }
+    throw e
+  }
+}
+
+const canOverwriteLogGetter = ({
+  completedExecutionLogMerging,
+  executionResult,
+}) => {
+  if (!completedExecutionLogMerging) {
+    return false
+  }
+  if (executionResult.status === "aborted") {
+    return true
+  }
+  if (executionResult.status !== "completed") {
+    return false
+  }
+  const { consoleCalls = [] } = executionResult
+  if (consoleCalls.length > 0) {
+    return false
+  }
+  return true
+}
+
+const executeInParallel = async ({
+  multipleExecutionsOperation,
+  maxExecutionsInParallel,
+  cooldownBetweenExecutions,
+  executionSteps,
+  start,
+}) => {
+  const executionResults = []
+  let progressionIndex = 0
+  let remainingExecutionCount = executionSteps.length
+
+  const nextChunk = async () => {
+    if (multipleExecutionsOperation.signal.aborted) {
+      return
+    }
+    const outputPromiseArray = []
+    while (
+      remainingExecutionCount > 0 &&
+      outputPromiseArray.length < maxExecutionsInParallel
+    ) {
+      remainingExecutionCount--
+      const outputPromise = executeOne(progressionIndex)
+      progressionIndex++
+      outputPromiseArray.push(outputPromise)
+    }
+    if (outputPromiseArray.length) {
+      await Promise.all(outputPromiseArray)
+      if (remainingExecutionCount > 0) {
+        await nextChunk()
+      }
+    }
+  }
+
+  const executeOne = async (index) => {
+    const input = executionSteps[index]
+    const output = await start(input)
+    if (!multipleExecutionsOperation.signal.aborted) {
+      executionResults[index] = output
+    }
+    if (cooldownBetweenExecutions) {
+      await new Promise((resolve) =>
+        setTimeout(resolve, cooldownBetweenExecutions),
+      )
+    }
+  }
+
+  await nextChunk()
+
+  return executionResults
+}
+
+const reportToSummary = (report) => {
+  const fileNames = Object.keys(report)
+  const countResultMatching = (predicate) => {
+    return fileNames.reduce((previous, fileName) => {
+      const fileExecutionResult = report[fileName]
+
+      return (
+        previous +
+        Object.keys(fileExecutionResult).filter((executionName) => {
+          const fileExecutionResultForRuntime =
+            fileExecutionResult[executionName]
+          return predicate(fileExecutionResultForRuntime)
+        }).length
+      )
+    }, 0)
+  }
+  const abortedCount = countResultMatching(({ status }) => status === "aborted")
+  const timedoutCount = countResultMatching(
+    ({ status }) => status === "timedout",
+  )
+  const erroredCount = countResultMatching(({ status }) => status === "errored")
+  const completedCount = countResultMatching(
+    ({ status }) => status === "completed",
+  )
+  return {
+    abortedCount,
+    timedoutCount,
+    erroredCount,
+    completedCount,
+  }
+}

package/src/internal/executing/{generateFileExecutionSteps.js → execution_steps.js}

@@ -1,5 +1,32 @@
+import { collectFiles } from "@jsenv/filesystem"
 import { createDetailedMessage } from "@jsenv/logger"
 
+export const generateExecutionSteps = async (
+  plan,
+  { signal, projectDirectoryUrl },
+) => {
+  const structuredMetaMap = {
+    filePlan: plan,
+  }
+
+  const fileResultArray = await collectFiles({
+    signal,
+    directoryUrl: projectDirectoryUrl,
+    structuredMetaMap,
+    predicate: ({ filePlan }) => filePlan,
+  })
+
+  const executionSteps = []
+  fileResultArray.forEach(({ relativeUrl, meta }) => {
+    const fileExecutionSteps = generateFileExecutionSteps({
+      fileRelativeUrl: relativeUrl,
+      filePlan: meta.filePlan,
+    })
+    executionSteps.push(...fileExecutionSteps)
+  })
+  return executionSteps
+}
+
 export const generateFileExecutionSteps = ({ fileRelativeUrl, filePlan }) => {
   const fileExecutionSteps = []
 

package/src/internal/executing/{launchAndExecute.js → launch_and_execute.js}

@@ -1,9 +1,9 @@
 import cuid from "cuid"
+import { resolveUrl, writeFile } from "@jsenv/filesystem"
 import { createLogger, createDetailedMessage } from "@jsenv/logger"
 import { Abort, raceCallbacks } from "@jsenv/abort"
-import { resolveUrl, writeFile } from "@jsenv/filesystem"
 
-import { composeTwoFileByFileIstanbulCoverages } from "./coverage_utils/istanbul_coverage_composition.js"
+import { composeTwoFileByFileIstanbulCoverages } from "@jsenv/core/src/internal/coverage/istanbul_coverage_composition.js"
 
 export const launchAndExecute = async ({
   signal = new AbortController().signal,

package/src/internal/executing/{executionLogs.js → logs_file_execution.js}

@@ -1,8 +1,9 @@
 import { ANSI, UNICODE } from "@jsenv/log"
 
-import { msAsDuration } from "../logs/msAsDuration.js"
+import { msAsDuration } from "@jsenv/core/src/internal/logs/ms_as_duration.js"
+
 import { EXECUTION_COLORS } from "./execution_colors.js"
-import { createIntermediateSummary } from "./createSummaryLog.js"
+import { createIntermediateSummary } from "./logs_plan_summary.js"
 
 export const formatExecuting = (
   { executionIndex },

package/src/internal/executing/{createSummaryLog.js → logs_plan_summary.js}

@@ -1,7 +1,8 @@
 import { ANSI } from "@jsenv/log"
 
-import { msAsDuration } from "../logs/msAsDuration.js"
-import { formatByte } from "../logs/byte.js"
+import { msAsDuration } from "@jsenv/core/src/internal/logs/ms_as_duration.js"
+import { formatByte } from "@jsenv/core/src/internal/logs/byte.js"
+
 import { EXECUTION_COLORS } from "./execution_colors.js"
 
 export const createSummaryLog = (

package/src/internal/features/browser_feature_detection/browser_feature_detection.js

@@ -1,4 +1,4 @@
-import { fetchJson } from "../../browser_utils/fetchJson.js"
+import { fetchJson } from "../../browser_utils/fetch_json.js"
 import { detectBrowser } from "../../browser_detection/browser_detection.js"
 
 import { supportsImportmap } from "./browser_feature_detect_importmap.js"

package/src/internal/features/features_compat_from_runtime.js

@@ -1,4 +1,4 @@
-import { findHighestVersion } from "@jsenv/core/src/internal/semantic_versioning/index.js"
+import { findHighestVersion } from "@jsenv/core/src/internal/semantic_versioning/highest_version.js"
 
 import { featuresCompatMap } from "./features_compatibility.js"
 

package/src/internal/features/node_feature_detection/node_feature_detection.js

@@ -1,4 +1,4 @@
-import { fetchSource } from "@jsenv/core/src/internal/node_runtime/fetchSource.js"
+import { fetchSource } from "@jsenv/core/src/internal/node_launcher/fetch_source.js"
 
 import { nodeSupportsDynamicImport } from "./node_feature_detect_dynamic_import.js"
 import { nodeSupportsTopLevelAwait } from "./node_feature_detect_top_level_await.js"