@jsenv/core 27.1.0 → 27.2.2

package/src/execute/runtimes/node/execute_using_dynamic_import.js

@@ -0,0 +1,49 @@
+import { startJsCoverage } from "./profiler_v8_coverage.js"
+import { startObservingPerformances } from "./node_execution_performance.js"
+
+export const executeUsingDynamicImport = async ({
+  rootDirectoryUrl,
+  fileUrl,
+  collectPerformance,
+  coverageEnabled,
+  coverageConfig,
+  coverageMethodForNodeJs,
+}) => {
+  let result = {}
+  const afterImportCallbacks = []
+  if (coverageEnabled && coverageMethodForNodeJs === "Profiler") {
+    const { filterV8Coverage } = await import(
+      "@jsenv/core/src/test/coverage/v8_coverage.js"
+    )
+    const { stopJsCoverage } = await startJsCoverage()
+    afterImportCallbacks.push(async () => {
+      const coverage = await stopJsCoverage()
+      const coverageLight = await filterV8Coverage(coverage, {
+        rootDirectoryUrl,
+        coverageConfig,
+      })
+      result.coverage = coverageLight
+    })
+  }
+  if (collectPerformance) {
+    const getPerformance = startObservingPerformances()
+    afterImportCallbacks.push(async () => {
+      const performance = await getPerformance()
+      result.performance = performance
+    })
+  }
+  const namespace = await import(fileUrl)
+  const namespaceResolved = {}
+  await Promise.all(
+    Object.keys(namespace).map(async (key) => {
+      const value = await namespace[key]
+      namespaceResolved[key] = value
+    }),
+  )
+  result.namespace = namespaceResolved
+  await afterImportCallbacks.reduce(async (previous, afterImportCallback) => {
+    await previous
+    await afterImportCallback()
+  }, Promise.resolve())
+  return result
+}

package/src/execute/runtimes/node/exit_codes.js

@@ -0,0 +1,9 @@
+// https://nodejs.org/api/process.html#process_signal_events
+const SIGINT_SIGNAL_NUMBER = 2
+const SIGABORT_SIGNAL_NUMBER = 6
+const SIGTERM_SIGNAL_NUMBER = 15
+export const EXIT_CODES = {
+  SIGINT: 128 + SIGINT_SIGNAL_NUMBER,
+  SIGABORT: 128 + SIGABORT_SIGNAL_NUMBER,
+  SIGTERM: 128 + SIGTERM_SIGNAL_NUMBER,
+}

package/src/execute/runtimes/node/{node_process.js → node_child_process.js}

@@ -1,29 +1,30 @@
 import { fork } from "node:child_process"
+import { fileURLToPath } from "node:url"
 import {
   Abort,
   raceCallbacks,
   createCallbackListNotifiedOnce,
 } from "@jsenv/abort"
-import { uneval } from "@jsenv/uneval"
-import { urlToFileSystemPath } from "@jsenv/urls"
 import { createDetailedMessage } from "@jsenv/log"
 import { memoize } from "@jsenv/utils/src/memoize/memoize.js"
 
 import { createChildExecOptions } from "./child_exec_options.js"
 import { ExecOptions } from "./exec_options.js"
 import { killProcessTree } from "./kill_process_tree.js"
+import { EXIT_CODES } from "./exit_codes.js"
 
-const NODE_CONTROLLABLE_FILE_URL = new URL(
-  "./controllable_file.mjs",
+const CONTROLLABLE_CHILD_PROCESS_URL = new URL(
+  "./controllable_child_process.mjs?entry_point",
   import.meta.url,
 ).href
 
-export const nodeProcess = {
-  name: "node",
+export const nodeChildProcess = {
+  type: "node",
+  name: "node_child_process",
   version: process.version.slice(1),
 }
 
-nodeProcess.run = async ({
+nodeChildProcess.run = async ({
   signal = new AbortController().signal,
   logger,
   logProcessCommand = false,
@@ -35,14 +36,15 @@ nodeProcess.run = async ({
   stopSignal,
   onConsole,
 
-  collectCoverage = false,
-  coverageForceIstanbul,
+  coverageEnabled = false,
+  coverageConfig,
+  coverageMethodForNodeJs,
   collectPerformance,
 
+  env,
   debugPort,
   debugMode,
   debugModeInheritBreak,
-  env,
   inheritProcessEnv = true,
   commandLineOptions = [],
   stdin = "pipe",
@@ -54,12 +56,9 @@ nodeProcess.run = async ({
   }
   env = {
     ...env,
-    COVERAGE_ENABLED: collectCoverage,
     JSENV: true,
   }
-  if (coverageForceIstanbul) {
-    // if we want to force istanbul, we will set process.env.NODE_V8_COVERAGE = ''
-    // into the child_process
+  if (coverageMethodForNodeJs !== "NODE_V8_COVERAGE") {
     env.NODE_V8_COVERAGE = ""
   }
   commandLineOptions = [
@@ -87,11 +86,11 @@ nodeProcess.run = async ({
     ...env,
   }
   logger[logProcessCommand ? "info" : "debug"](
-    `${process.argv[0]} ${execArgv.join(" ")} ${urlToFileSystemPath(
-      NODE_CONTROLLABLE_FILE_URL,
+    `${process.argv[0]} ${execArgv.join(" ")} ${fileURLToPath(
+      CONTROLLABLE_CHILD_PROCESS_URL,
     )}`,
   )
-  const childProcess = fork(urlToFileSystemPath(NODE_CONTROLLABLE_FILE_URL), {
+  const childProcess = fork(fileURLToPath(CONTROLLABLE_CHILD_PROCESS_URL), {
     execArgv,
     // silent: true
     stdio: ["pipe", "pipe", "pipe", "ipc"],
@@ -114,9 +113,9 @@ nodeProcess.run = async ({
     childProcess.stderr.pipe(stderr)
   }
   const childProcessReadyPromise = new Promise((resolve) => {
-    onceProcessMessage(childProcess, "ready", resolve)
+    onceChildProcessMessage(childProcess, "ready", resolve)
   })
-  const removeOutputListener = installProcessOutputListener(
+  const removeOutputListener = installChildProcessOutputListener(
     childProcess,
     ({ type, text }) => {
       onConsole({ type, text })
@@ -171,15 +170,15 @@ nodeProcess.run = async ({
         // },
         // https://nodejs.org/api/child_process.html#child_process_event_error
         error: (cb) => {
-          return onceProcessEvent(childProcess, "error", cb)
+          return onceChildProcessEvent(childProcess, "error", cb)
         },
         exit: (cb) => {
-          return onceProcessEvent(childProcess, "exit", (code, signal) => {
+          return onceChildProcessEvent(childProcess, "exit", (code, signal) => {
             cb({ code, signal })
           })
         },
         response: (cb) => {
-          onceProcessMessage(childProcess, "action-result", cb)
+          return onceChildProcessMessage(childProcess, "action-result", cb)
         },
       },
       resolve,
@@ -189,11 +188,19 @@ nodeProcess.run = async ({
     actionOperation.throwIfAborted()
     await childProcessReadyPromise
     actionOperation.throwIfAborted()
-    await sendToProcess(childProcess, "action", {
-      actionType: "execute-using-dynamic-import",
-      actionParams: {
-        fileUrl: new URL(fileRelativeUrl, rootDirectoryUrl).href,
-        collectPerformance,
+    await sendToChildProcess(childProcess, {
+      type: "action",
+      data: {
+        actionType: "execute-using-dynamic-import",
+        actionParams: {
+          rootDirectoryUrl,
+          fileUrl: new URL(fileRelativeUrl, rootDirectoryUrl).href,
+          collectPerformance,
+          coverageEnabled,
+          coverageConfig,
+          coverageMethodForNodeJs,
+          exitAfterAction: true,
+        },
       },
     })
     const winner = await winnerPromise
@@ -224,9 +231,9 @@ nodeProcess.run = async ({
       if (
         code === null ||
         code === 0 ||
-        code === SIGINT_EXIT_CODE ||
-        code === SIGTERM_EXIT_CODE ||
-        code === SIGABORT_EXIT_CODE
+        code === EXIT_CODES.SIGINT ||
+        code === EXIT_CODES.SIGTERM ||
+        code === EXIT_CODES.SIGABORT
       ) {
         return {
           status: "errored",
@@ -275,13 +282,6 @@ nodeProcess.run = async ({
   return result
 }
 
-// https://nodejs.org/api/process.html#process_signal_events
-const SIGINT_SIGNAL_NUMBER = 2
-const SIGABORT_SIGNAL_NUMBER = 6
-const SIGTERM_SIGNAL_NUMBER = 15
-const SIGINT_EXIT_CODE = 128 + SIGINT_SIGNAL_NUMBER
-const SIGABORT_EXIT_CODE = 128 + SIGABORT_SIGNAL_NUMBER
-const SIGTERM_EXIT_CODE = 128 + SIGTERM_SIGNAL_NUMBER
 // http://man7.org/linux/man-pages/man7/signal.7.html
 // https:// github.com/nodejs/node/blob/1d9511127c419ec116b3ddf5fc7a59e8f0f1c1e4/lib/internal/child_process.js#L472
 const GRACEFUL_STOP_SIGNAL = "SIGTERM"
@@ -290,20 +290,26 @@ const STOP_SIGNAL = "SIGKILL"
 // but I'm not sure and it changes nothing so just use SIGKILL
 const GRACEFUL_STOP_FAILED_SIGNAL = "SIGKILL"
 
-const sendToProcess = async (childProcess, type, data) => {
-  const source = uneval(data, { functionAllowed: true })
+const sendToChildProcess = async (childProcess, { type, data }) => {
   return new Promise((resolve, reject) => {
-    childProcess.send({ type, data: source }, (error) => {
-      if (error) {
-        reject(error)
-      } else {
-        resolve()
-      }
-    })
+    childProcess.send(
+      {
+        jsenv: true,
+        type,
+        data,
+      },
+      (error) => {
+        if (error) {
+          reject(error)
+        } else {
+          resolve()
+        }
+      },
+    )
   })
 }
 
-const installProcessOutputListener = (childProcess, callback) => {
+const installChildProcessOutputListener = (childProcess, callback) => {
   // beware that we may receive ansi output here, should not be a problem but keep that in mind
   const stdoutDataCallback = (chunk) => {
     callback({ type: "log", text: String(chunk) })
@@ -319,9 +325,9 @@ const installProcessOutputListener = (childProcess, callback) => {
   }
 }
 
-const onceProcessMessage = (childProcess, type, callback) => {
+const onceChildProcessMessage = (childProcess, type, callback) => {
   const onmessage = (message) => {
-    if (message.type === type) {
+    if (message && message.jsenv && message.type === type) {
       childProcess.removeListener("message", onmessage)
       // eslint-disable-next-line no-eval
       callback(message.data ? eval(`(${message.data})`) : "")
@@ -333,7 +339,7 @@ const onceProcessMessage = (childProcess, type, callback) => {
   }
 }
 
-const onceProcessEvent = (childProcess, type, callback) => {
+const onceChildProcessEvent = (childProcess, type, callback) => {
   childProcess.once(type, callback)
   return () => {
     childProcess.removeListener(type, callback)

package/src/execute/runtimes/node/node_worker_thread.js

@@ -1,28 +1,271 @@
 // https://github.com/avajs/ava/blob/576f534b345259055c95fa0c2b33bef10847a2af/lib/fork.js#L23
 // https://nodejs.org/api/worker_threads.html
 // https://github.com/avajs/ava/blob/576f534b345259055c95fa0c2b33bef10847a2af/lib/worker/base.js
-// import { Worker } from "node:worker_threads"
-
-// export const nodeWorkerThread = {
-//   name: "node_worker",
-//   version: process.version.slice(1),
-//   launch: async ({ env }) => {
-//     // https://nodejs.org/api/worker_threads.html#new-workerfilename-options
-//     const worker = new Worker(workerPath, {
-//       argv: options.workerArgv,
-//       env: {
-//         NODE_ENV: "test",
-//         JSENV: true,
-//         ...process.env,
-//       },
-//       execArgv: [...execArgv, ...additionalExecArgv],
-//       workerData: {
-//         options,
-//       },
-//       trackUnmanagedFds: true,
-//       stdin: true,
-//       stdout: true,
-//       stderr: true,
-//     })
-//   },
-// }
+import { Worker } from "node:worker_threads"
+import { fileURLToPath } from "node:url"
+import {
+  Abort,
+  createCallbackListNotifiedOnce,
+  raceCallbacks,
+} from "@jsenv/abort"
+import { memoize } from "@jsenv/utils/src/memoize/memoize.js"
+
+import { createChildExecOptions } from "./child_exec_options.js"
+import { ExecOptions } from "./exec_options.js"
+import { EXIT_CODES } from "./exit_codes.js"
+
+const CONTROLLABLE_WORKER_THREAD_URL = new URL(
+  "./controllable_worker_thread.mjs?entry_point",
+  import.meta.url,
+).href
+
+export const nodeWorkerThread = {
+  type: "node",
+  name: "node_worker_thread",
+  version: process.version.slice(1),
+}
+
+nodeWorkerThread.run = async ({
+  signal = new AbortController().signal,
+  // logger,
+  rootDirectoryUrl,
+  fileRelativeUrl,
+
+  keepRunning,
+  stopSignal,
+  onConsole,
+
+  coverageConfig,
+  coverageMethodForNodeJs,
+  coverageEnabled = false,
+  collectPerformance,
+
+  env,
+  debugPort,
+  debugMode,
+  debugModeInheritBreak,
+  inheritProcessEnv = true,
+  commandLineOptions = [],
+}) => {
+  if (env !== undefined && typeof env !== "object") {
+    throw new TypeError(`env must be an object, got ${env}`)
+  }
+  env = {
+    ...env,
+    JSENV: true,
+  }
+  if (coverageMethodForNodeJs !== "NODE_V8_COVERAGE") {
+    env.NODE_V8_COVERAGE = ""
+  }
+
+  const workerThreadExecOptions = await createChildExecOptions({
+    signal,
+    debugPort,
+    debugMode,
+    debugModeInheritBreak,
+  })
+  const execArgvForWorkerThread = ExecOptions.toExecArgv({
+    ...workerThreadExecOptions,
+    ...ExecOptions.fromExecArgv(commandLineOptions),
+  })
+  const envForWorkerThread = {
+    ...(inheritProcessEnv ? process.env : {}),
+    ...env,
+  }
+
+  const cleanupCallbackList = createCallbackListNotifiedOnce()
+  const cleanup = async (reason) => {
+    await cleanupCallbackList.notify({ reason })
+  }
+  const actionOperation = Abort.startOperation()
+  actionOperation.addAbortSignal(signal)
+  // https://nodejs.org/api/worker_threads.html#new-workerfilename-options
+  const workerThread = new Worker(
+    fileURLToPath(CONTROLLABLE_WORKER_THREAD_URL),
+    {
+      env: envForWorkerThread,
+      execArgv: execArgvForWorkerThread,
+      // workerData: { options },
+      // trackUnmanagedFds: true,
+      stdin: true,
+      stdout: true,
+      stderr: true,
+    },
+  )
+  const removeOutputListener = installWorkerThreadOutputListener(
+    workerThread,
+    ({ type, text }) => {
+      onConsole({ type, text })
+    },
+  )
+  const workerThreadReadyPromise = new Promise((resolve) => {
+    onceWorkerThreadMessage(workerThread, "ready", resolve)
+  })
+
+  const stop = memoize(async () => {
+    await workerThreadReadyPromise
+    await workerThread.terminate()
+  })
+
+  const winnerPromise = new Promise((resolve) => {
+    raceCallbacks(
+      {
+        aborted: (cb) => {
+          return actionOperation.addAbortCallback(cb)
+        },
+        error: (cb) => {
+          return onceWorkerThreadEvent(workerThread, "error", cb)
+        },
+        exit: (cb) => {
+          return onceWorkerThreadEvent(workerThread, "exit", (code, signal) => {
+            cb({ code, signal })
+          })
+        },
+        response: (cb) => {
+          return onceWorkerThreadMessage(workerThread, "action-result", cb)
+        },
+      },
+      resolve,
+    )
+  })
+
+  const getResult = async () => {
+    actionOperation.throwIfAborted()
+    await workerThreadReadyPromise
+    actionOperation.throwIfAborted()
+    await sendToWorkerThread(workerThread, {
+      type: "action",
+      data: {
+        actionType: "execute-using-dynamic-import",
+        actionParams: {
+          rootDirectoryUrl,
+          fileUrl: new URL(fileRelativeUrl, rootDirectoryUrl).href,
+          collectPerformance,
+          coverageEnabled,
+          coverageConfig,
+          coverageMethodForNodeJs,
+          exitAfterAction: true,
+        },
+      },
+    })
+    const winner = await winnerPromise
+    if (winner.name === "aborted") {
+      return {
+        status: "aborted",
+      }
+    }
+    if (winner.name === "error") {
+      const error = winner.data
+      removeOutputListener()
+      return {
+        status: "errored",
+        error,
+      }
+    }
+    if (winner.name === "exit") {
+      const { code } = winner.data
+      await cleanup("process exit")
+      if (code === 12) {
+        return {
+          status: "errored",
+          error: new Error(
+            `node process exited with 12 (the forked child process wanted to use a non-available port for debug)`,
+          ),
+        }
+      }
+      if (
+        code === null ||
+        code === 0 ||
+        code === EXIT_CODES.SIGINT ||
+        code === EXIT_CODES.SIGTERM ||
+        code === EXIT_CODES.SIGABORT
+      ) {
+        return {
+          status: "errored",
+          error: new Error(`node worker thread exited during execution`),
+        }
+      }
+      // process.exit(1) in child process or process.exitCode = 1 + process.exit()
+      // means there was an error even if we don't know exactly what.
+      return {
+        status: "errored",
+        error: new Error(
+          `node worker thread exited with code ${code} during execution`,
+        ),
+      }
+    }
+    const { status, value } = winner.data
+    if (status === "action-failed") {
+      return {
+        status: "errored",
+        error: value,
+      }
+    }
+    const { namespace, performance, coverage } = value
+    return {
+      status: "completed",
+      namespace,
+      performance,
+      coverage,
+    }
+  }
+
+  let result
+  try {
+    result = await getResult()
+  } catch (e) {
+    result = {
+      status: "errored",
+      error: e,
+    }
+  }
+
+  if (keepRunning) {
+    stopSignal.notify = stop
+  } else {
+    await stop()
+  }
+  await actionOperation.end()
+  return result
+}
+
+const installWorkerThreadOutputListener = (workerThread, callback) => {
+  // beware that we may receive ansi output here, should not be a problem but keep that in mind
+  const stdoutDataCallback = (chunk) => {
+    callback({ type: "log", text: String(chunk) })
+  }
+  workerThread.stdout.on("data", stdoutDataCallback)
+  const stdErrorDataCallback = (chunk) => {
+    callback({ type: "error", text: String(chunk) })
+  }
+  workerThread.stderr.on("data", stdErrorDataCallback)
+  return () => {
+    workerThread.stdout.removeListener("data", stdoutDataCallback)
+    workerThread.stderr.removeListener("data", stdoutDataCallback)
+  }
+}
+
+const sendToWorkerThread = (worker, { type, data }) => {
+  worker.postMessage({ jsenv: true, type, data })
+}
+
+const onceWorkerThreadMessage = (workerThread, type, callback) => {
+  const onmessage = (message) => {
+    if (message && message.jsenv && message.type === type) {
+      workerThread.removeListener("message", onmessage)
+      // eslint-disable-next-line no-eval
+      callback(message.data ? eval(`(${message.data})`) : undefined)
+    }
+  }
+  workerThread.on("message", onmessage)
+  return () => {
+    workerThread.removeListener("message", onmessage)
+  }
+}
+
+const onceWorkerThreadEvent = (worker, type, callback) => {
+  worker.once(type, callback)
+  return () => {
+    worker.removeListener(type, callback)
+  }
+}

package/src/execute/runtimes/node/profiler_v8_coverage.js

@@ -0,0 +1,56 @@
+/*
+ * Calling Profiler.startPreciseCoverage DO NOT propagate to
+ * subprocesses (new Worker or child_process.fork())
+ * So the best solution remains NODE_V8_COVERAGE
+ * This profiler strategy remains useful when:
+ * - As fallback when NODE_V8_COVERAGE is not configured
+ * - If explicitely enabled with coverageMethodForNodeJs: 'Profiler'
+ *   - Used by jsenv during automated tests about coverage
+ *   - Anyone prefering this approach over NODE_V8_COVERAGE and assuming
+ *     it will not fork subprocess or don't care if coverage is missed for this code
+ * - https://v8.dev/blog/javascript-code-coverage#for-embedders
+ * - https://github.com/nodejs/node/issues/28283
+ * - https://vanilla.aslushnikov.com/?Profiler.startPreciseCoverage
+ */
+
+import { Session } from "node:inspector"
+
+export const startJsCoverage = async ({
+  callCount = true,
+  detailed = true,
+} = {}) => {
+  const session = new Session()
+  session.connect()
+  const postSession = (action, options) => {
+    const promise = new Promise((resolve, reject) => {
+      session.post(action, options, (error, data) => {
+        if (error) {
+          reject(error)
+        } else {
+          resolve(data)
+        }
+      })
+    })
+    return promise
+  }
+
+  await postSession("Profiler.enable")
+  await postSession("Profiler.startPreciseCoverage", { callCount, detailed })
+
+  const takeJsCoverage = async () => {
+    const coverage = await postSession("Profiler.takePreciseCoverage")
+    return coverage
+  }
+
+  const stopJsCoverage = async () => {
+    const coverage = await takeJsCoverage()
+    await postSession("Profiler.stopPreciseCoverage")
+    await postSession("Profiler.disable")
+    return coverage
+  }
+
+  return {
+    takeJsCoverage,
+    stopJsCoverage,
+  }
+}

package/src/main.js

@@ -2,6 +2,7 @@
 export { startDevServer } from "./dev/start_dev_server.js"
 // test
 export { executeTestPlan } from "./test/execute_test_plan.js"
+// runtimes (used to execute tests)
 export {
   chromium,
   chromiumIsolatedTab,
@@ -14,7 +15,8 @@ export {
   webkit,
   webkitIsolatedTab,
 } from "./execute/runtimes/browsers/webkit.js"
-export { nodeProcess } from "./execute/runtimes/node/node_process.js"
+export { nodeChildProcess } from "./execute/runtimes/node/node_child_process.js"
+export { nodeWorkerThread } from "./execute/runtimes/node/node_worker_thread.js"
 // build
 export { build } from "./build/build.js"
 export { startBuildServer } from "./build/start_build_server.js"