@jsenv/test 3.2.0 → 3.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/test",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -38,13 +38,13 @@
   "dependencies": {
     "@c88/v8-coverage": "0.1.1",
     "@jsenv/abort": "4.3.0",
-    "@jsenv/ast": "6.1.0",
-    "@jsenv/filesystem": "4.7.2",
-    "@jsenv/github-check-run": "0.1.16",
-    "@jsenv/humanize": "1.1.3",
-    "@jsenv/plugin-supervisor": "1.4.12",
+    "@jsenv/ast": "6.1.2",
+    "@jsenv/filesystem": "4.7.3",
+    "@jsenv/github-check-run": "0.1.18",
+    "@jsenv/humanize": "1.2.0",
+    "@jsenv/plugin-supervisor": "1.4.14",
     "@jsenv/url-meta": "8.4.2",
-    "@jsenv/urls": "2.2.7",
+    "@jsenv/urls": "2.2.8",
     "@jsenv/utils": "2.1.1",
     "istanbul-lib-coverage": "3.2.2",
     "istanbul-lib-instrument": "6.0.3",

package/src/execution/exception.js

@@ -211,9 +211,7 @@ const getErrorName = (value) => {
 
 const getPropertiesFromEvalOrigin = (origin) => {
   // Most eval() calls are in this format
-  const topLevelEvalMatch = /^eval at ([^(]+) \((.+):(\d+):(\d+)\)$/.exec(
-    origin,
-  );
+  const topLevelEvalMatch = /^eval at [^(]+ \(.+:\d+:\d+\)$/.exec(origin);
   if (topLevelEvalMatch) {
     const source = topLevelEvalMatch[2];
     const line = Number(topLevelEvalMatch[3]);
@@ -225,7 +223,7 @@ const getPropertiesFromEvalOrigin = (origin) => {
     };
   }
   // Parse nested eval() calls using recursion
-  const nestedEvalMatch = /^eval at ([^(]+) \((.+)\)$/.exec(origin);
+  const nestedEvalMatch = /^eval at [^(]+ \(.+\)$/.exec(origin);
   if (nestedEvalMatch) {
     return getPropertiesFromEvalOrigin(nestedEvalMatch[2]);
   }

package/src/execution/execute_test_plan.js

@@ -33,6 +33,7 @@ import { assertAndNormalizeWebServer } from "./web_server_param.js";
 import { githubAnnotationFromError } from "./github_annotation_from_error.js";
 import { run } from "./run.js";
 import { reporterList, renderOutroContent } from "./reporters/reporter_list.js";
+import { createIsInsideFragment } from "./is_inside_fragment.js";
 
 /**
  * Execute a list of files and log how it goes.
@@ -114,6 +115,11 @@ export const executeTestPlan = async ({
   handleSIGTERM = true,
   updateProcessExitCode = true,
   parallel = parallelDefault,
+  // https://github.com/avajs/ava/blob/main/docs/recipes/splitting-tests-ci.md
+  // https://playwright.dev/docs/test-sharding
+  fragment,
+  fragmentByRuntime,
+  fragmentBy,
   defaultMsAllocatedPerExecution = 30_000,
   failFast = false,
   // keepRunning: false to ensure runtime is stopped once executed
@@ -207,6 +213,9 @@ export const executeTestPlan = async ({
     rootDirectoryUrl: String(rootDirectoryUrl),
     patterns: Object.keys(testPlan),
     groups: {},
+    fragment,
+    fragmentStart: undefined,
+    fragmentEnd: undefined,
     counters: {
       planified: 0,
       remaining: 0,
@@ -214,6 +223,7 @@ export const executeTestPlan = async ({
       executing: 0,
       executed: 0,
 
+      skipped: 0,
       aborted: 0,
       cancelled: 0,
       timedout: 0,
@@ -243,6 +253,36 @@ export const executeTestPlan = async ({
   };
   const beforeEachCallbackSet = new Set();
   const beforeEachInOrderCallbackSet = new Set();
+  const triggerBeforeEach = (execution, testPlanResult, testPlanHelpers) => {
+    for (const beforeEachCallback of beforeEachCallbackSet) {
+      const returnValue = beforeEachCallback(
+        execution,
+        testPlanResult,
+        testPlanHelpers,
+      );
+      if (typeof returnValue === "function") {
+        const callback = (...args) => {
+          afterEachCallbackSet.delete(callback);
+          return returnValue(...args);
+        };
+        afterEachCallbackSet.add(callback);
+      }
+    }
+    for (const beforeEachInOrderCallback of beforeEachInOrderCallbackSet) {
+      const returnValue = beforeEachInOrderCallback(
+        execution,
+        testPlanResult,
+        testPlanHelpers,
+      );
+      if (typeof returnValue === "function") {
+        const callback = (...args) => {
+          afterEachInOrderCallbackSet.delete(callback);
+          return returnValue(...args);
+        };
+        afterEachInOrderCallbackSet.add(callback);
+      }
+    }
+  };
   const afterEachCallbackSet = new Set();
   const afterEachInOrderCallbackSet = new Set();
   const afterAllCallbackSet = new Set();
@@ -375,6 +415,16 @@ export const executeTestPlan = async ({
           );
         }
       }
+      // fragment/fragmentByRuntime
+      {
+        if (fragment) {
+          if (!/[0-9]+\/[0-9]+/.test(fragment)) {
+            throw new TypeError(
+              `fragment must look like number/number, received ${fragment}`,
+            );
+          }
+        }
+      }
       // testPlan
       {
         if (typeof testPlan !== "object") {
@@ -579,9 +629,21 @@ To fix this warning:
       }
     }
 
-    const executionPlanifiedSet = new Set();
+    const executionPlanifiedArray = [];
+    const getPreviousExecution = (execution) => {
+      const index = execution.index;
+      return executionPlanifiedArray[index - 1] || null;
+    };
+    const getNextExecution = (execution) => {
+      const index = execution.index;
+      return executionPlanifiedArray[index + 1] || null;
+    };
+    const testPlanHelpers = {
+      getPreviousExecution,
+      getNextExecution,
+    };
 
-    // collect files to execute + fill executionPlanifiedSet
+    // collect files to execute + fill executionPlanifiedArray
     {
       const fileResultArray = await collectFiles({
         signal,
@@ -589,6 +651,11 @@ To fix this warning:
         associations: { testPlan },
         predicate: ({ testPlan }) => testPlan,
       });
+
+      // idéalement dans les logs on aurait un truc comme ceci:
+      // --- 100 execution found ---
+      // Executing only from 33 to 66 according to fragment: "2/3"
+
       let index = 0;
       let lastExecution;
       const fileExecutionCountMap = new Map();
@@ -611,10 +678,6 @@ To fix this warning:
               ),
             );
           }
-          if (stepConfig.runtime?.disabled) {
-            continue;
-          }
-
           const {
             runtime,
             runtimeParams,
@@ -671,6 +734,8 @@ To fix this warning:
             runtimeName,
             runtimeVersion,
             params,
+            skipped: false,
+            skipReason: "",
 
             // will be set by run()
             status: "planified",
@@ -681,7 +746,7 @@ To fix this warning:
           }
 
           lastExecution = execution;
-          executionPlanifiedSet.add(execution);
+          executionPlanifiedArray.push(execution);
           const existingResults = results[relativeUrl];
           if (existingResults) {
             existingResults[groupName] = execution.result;
@@ -701,9 +766,48 @@ To fix this warning:
               runtimeVersion,
             };
           }
+
+          if (runtime?.disabled) {
+            execution.skipped = true;
+            execution.skipReason =
+              runtime.disabledReason || "reason not specified";
+          } else if (fragmentByRuntime && runtime.name !== fragmentByRuntime) {
+            execution.skipped = true;
+            execution.skipReason = "runtime disabled";
+          } else if (fragmentBy) {
+            const fragmentByResult = fragmentBy(execution);
+            if (fragmentByResult) {
+              execution.skipped = true;
+              execution.skipReason =
+                typeof fragmentByResult === "string"
+                  ? fragmentByResult
+                  : `reason not specified`;
+            }
+          }
           index++;
         }
       }
+
+      if (fragment) {
+        const total = executionPlanifiedArray.length;
+        const isInsideFragment = createIsInsideFragment(fragment, total);
+        for (const execution of executionPlanifiedArray) {
+          if (isInsideFragment(execution.index)) {
+            const executionNumber = execution.index + 1;
+            if (testPlanResult.fragmentStart === undefined) {
+              testPlanResult.fragmentStart = executionNumber;
+            }
+            if (
+              testPlanResult.fragmentEnd === undefined ||
+              executionNumber > testPlanResult.fragmentEnd
+            ) {
+              testPlanResult.fragmentEnd = executionNumber;
+            }
+          } else {
+            execution.skipped = true;
+          }
+        }
+      }
       fileResultArray.length = 0;
       fileExecutionCountMap.clear();
       if (lastExecution) {
@@ -714,11 +818,11 @@ To fix this warning:
     counters.planified =
       counters.remaining =
       counters.waiting =
-        executionPlanifiedSet.size;
+        executionPlanifiedArray.length;
     countersInOrder.planified =
       countersInOrder.remaining =
       countersInOrder.waiting =
-        executionPlanifiedSet.size;
+        executionPlanifiedArray.length;
     if (githubCheck) {
       const githubCheckRun = await startGithubCheckRun({
         logLevel: githubCheck.logLevel,
@@ -728,7 +832,7 @@ To fix this warning:
         commitSha: githubCheck.commitSha,
         checkName: githubCheck.name,
         checkTitle: githubCheck.title,
-        checkSummary: `${executionPlanifiedSet.size} files will be executed`,
+        checkSummary: `${executionPlanifiedArray.length} files will be executed`,
       });
       const annotations = [];
       reporters.push({
@@ -808,80 +912,66 @@ To fix this warning:
 
       const callWhenPreviousExecutionAreDone = createCallOrderer();
 
-      const executionRemainingSet = new Set(executionPlanifiedSet);
+      const executionRemainingSet = new Set(executionPlanifiedArray);
       const executionExecutingSet = new Set();
       const usedTagSet = new Set();
       const start = async (execution) => {
-        if (execution.params.uses) {
-          for (const tagThatWillBeUsed of execution.params.uses) {
-            usedTagSet.add(tagThatWillBeUsed);
-          }
-        }
         execution.fileExecutionCount = Object.keys(
           testPlanResult.results[execution.fileRelativeUrl],
         ).length;
         mutateCountersBeforeExecutionStarts(counters, execution);
         mutateCountersBeforeExecutionStarts(countersInOrder, execution);
-
-        execution.status = "executing";
         executionRemainingSet.delete(execution);
         executionExecutingSet.add(execution);
-        for (const beforeEachCallback of beforeEachCallbackSet) {
-          const returnValue = beforeEachCallback(execution, testPlanResult);
-          if (typeof returnValue === "function") {
-            const callback = (...args) => {
-              afterEachCallbackSet.delete(callback);
-              return returnValue(...args);
-            };
-            afterEachCallbackSet.add(callback);
+        triggerBeforeEach(execution, testPlanResult, testPlanHelpers);
+        if (execution.skipped) {
+          execution.result.status = "skipped";
+          execution.result.value = execution.skipReason;
+        } else {
+          if (execution.params.uses) {
+            for (const tagThatWillBeUsed of execution.params.uses) {
+              usedTagSet.add(tagThatWillBeUsed);
+            }
           }
-        }
-        for (const beforeEachInOrderCallback of beforeEachInOrderCallbackSet) {
-          const returnValue = beforeEachInOrderCallback(
-            execution,
-            testPlanResult,
-          );
-          if (typeof returnValue === "function") {
-            const callback = (...args) => {
-              afterEachInOrderCallbackSet.delete(callback);
-              return returnValue(...args);
-            };
-            afterEachInOrderCallbackSet.add(callback);
+          execution.status = "executing";
+          const executionResult = await run({
+            ...execution.params,
+            signal: operation.signal,
+            logger,
+            keepRunning,
+            mirrorConsole: false, // might be executed in parallel: log would be a mess to read
+            coverageEnabled: Boolean(coverage),
+            coverageTempDirectoryUrl: coverage?.tempDirectoryUrl,
+          });
+          Object.assign(execution.result, executionResult);
+          execution.status = "executed";
+          if (execution.params.uses) {
+            for (const tagNoLongerInUse of execution.params.uses) {
+              usedTagSet.delete(tagNoLongerInUse);
+            }
           }
-        }
-        const executionResult = await run({
-          ...execution.params,
-          signal: operation.signal,
-          logger,
-          keepRunning,
-          mirrorConsole: false, // might be executed in parallel: log would be a mess to read
-          coverageEnabled: Boolean(coverage),
-          coverageTempDirectoryUrl: coverage?.tempDirectoryUrl,
-        });
-        Object.assign(execution.result, executionResult);
-        execution.status = "executed";
-        executionExecutingSet.delete(execution);
-        mutateCountersAfterExecutionEnds(counters, execution);
-        if (execution.result.status !== "completed") {
-          testPlanResult.failed = true;
-          if (updateProcessExitCode) {
-            process.exitCode = 1;
+          if (execution.result.status !== "completed") {
+            testPlanResult.failed = true;
+            if (updateProcessExitCode) {
+              process.exitCode = 1;
+            }
           }
         }
+        mutateCountersAfterExecutionEnds(counters, execution);
+        executionExecutingSet.delete(execution);
         for (const afterEachCallback of afterEachCallbackSet) {
-          afterEachCallback(execution, testPlanResult);
+          afterEachCallback(execution, testPlanResult, testPlanHelpers);
         }
         callWhenPreviousExecutionAreDone(execution.index, () => {
           mutateCountersAfterExecutionEnds(countersInOrder, execution);
           for (const afterEachInOrderCallback of afterEachInOrderCallbackSet) {
-            afterEachInOrderCallback(execution, testPlanResult);
+            afterEachInOrderCallback(
+              execution,
+              testPlanResult,
+              testPlanHelpers,
+            );
           }
         });
-        if (execution.params.uses) {
-          for (const tagNoLongerInUse of execution.params.uses) {
-            usedTagSet.delete(tagNoLongerInUse);
-          }
-        }
         if (testPlanResult.failed && failFast && counters.remaining) {
           logger.info(`"failFast" enabled -> cancel remaining executions`);
           failFastAbortController.abort();
@@ -1096,7 +1186,9 @@ const mutateCountersAfterExecutionEnds = (counters, execution) => {
   counters.executing--;
   counters.executed++;
   counters.remaining--;
-  if (execution.result.status === "aborted") {
+  if (execution.result.status === "skipped") {
+    counters.skipped++;
+  } else if (execution.result.status === "aborted") {
     counters.aborted++;
   } else if (execution.result.status === "timedout") {
     counters.timedout++;

package/src/execution/format_error_for_terminal.js

@@ -2,6 +2,8 @@ import { readFileSync } from "node:fs";
 import { generateContentFrame, ANSI } from "@jsenv/humanize";
 import { urlToFileSystemPath } from "@jsenv/urls";
 
+import { replaceUrls } from "./replace_urls.js";
+
 export const formatErrorForTerminal = (
   error,
   { rootDirectoryUrl, mainFileRelativeUrl, mockFluctuatingValues, tryColors },
@@ -133,54 +135,3 @@ export const formatErrorForTerminal = (
 
   return text;
 };
-
-// `Error: yo
-// at Object.execute (http://127.0.0.1:57300/build/src/__test__/file-throw.js:9:13)
-// at doExec (http://127.0.0.1:3000/src/__test__/file-throw.js:452:38)
-// at postOrderExec (http://127.0.0.1:3000/src/__test__/file-throw.js:448:16)
-// at http://127.0.0.1:3000/src/__test__/file-throw.js:399:18`.replace(/(?:https?|ftp|file):\/\/(.*+)$/gm, (...args) => {
-//   debugger
-// })
-const replaceUrls = (source, replace) => {
-  return source.replace(/(?:https?|ftp|file):\/\/\S+/gm, (match) => {
-    let replacement = "";
-    const lastChar = match[match.length - 1];
-
-    // hotfix because our url regex sucks a bit
-    const endsWithSeparationChar = lastChar === ")" || lastChar === ":";
-    if (endsWithSeparationChar) {
-      match = match.slice(0, -1);
-    }
-
-    const lineAndColumnPattern = /:([0-9]+):([0-9]+)$/;
-    const lineAndColumMatch = match.match(lineAndColumnPattern);
-    if (lineAndColumMatch) {
-      const lineAndColumnString = lineAndColumMatch[0];
-      const lineString = lineAndColumMatch[1];
-      const columnString = lineAndColumMatch[2];
-      replacement = replace({
-        url: match.slice(0, -lineAndColumnString.length),
-        line: lineString ? parseInt(lineString) : null,
-        column: columnString ? parseInt(columnString) : null,
-      });
-    } else {
-      const linePattern = /:([0-9]+)$/;
-      const lineMatch = match.match(linePattern);
-      if (lineMatch) {
-        const lineString = lineMatch[0];
-        replacement = replace({
-          url: match.slice(0, -lineString.length),
-          line: lineString ? parseInt(lineString) : null,
-        });
-      } else {
-        replacement = replace({
-          url: match,
-        });
-      }
-    }
-    if (endsWithSeparationChar) {
-      return `${replacement}${lastChar}`;
-    }
-    return replacement;
-  });
-};

package/src/execution/github_annotation_from_error.js

@@ -85,7 +85,7 @@ const stringifyUrlSite = ({ url, line, column }) => {
 };
 
 const replaceUrls = (source, replace) => {
-  return source.replace(/(?:https?|ftp|file):\/\/\S+/gm, (match) => {
+  return source.replace(/(?:https?|ftp|file):\/\/\S+/g, (match) => {
     let replacement = "";
     const lastChar = match[match.length - 1];
 

package/src/execution/is_inside_fragment.js

@@ -0,0 +1,32 @@
+export const createIsInsideFragment = (fragment, total) => {
+  let [dividend, divisor] = fragment.split("/");
+  dividend = parseInt(dividend);
+  divisor = parseInt(divisor);
+  const groupSize = Math.ceil(total / divisor);
+  let from;
+  let to;
+  if (groupSize === 0) {
+    if (dividend === 1) {
+      from = 0;
+      to = 0;
+    } else {
+      from = Infinity;
+      to = -1;
+    }
+  } else if (dividend === 1) {
+    from = 0;
+    to = groupSize;
+  } else {
+    from = (dividend - 1) * groupSize;
+    if (dividend === divisor) {
+      to = total;
+    } else {
+      to = from + groupSize;
+    }
+  }
+  return (index) => {
+    if (index < from) return false;
+    if (index >= to) return false;
+    return true;
+  };
+};

package/src/execution/junit_xml_file/report_as_junit_xml.js

@@ -46,22 +46,26 @@ export const reportAsJunitXml = async (
       });
       testSuite.appendChild(testCase);
 
-      if (executionResult.status === "aborted") {
+      if (executionResult.status === "skipped") {
+        testCase.attributes.skipped = 1;
+        const skipped = testCase.createNode("skipped", {
+          message: "Execution was skipped",
+        });
+        testSuite.appendChild(skipped);
+      } else if (executionResult.status === "aborted") {
         testCase.attributes.skipped = 1;
         const skipped = testCase.createNode("skipped", {
           message: "Execution was aborted",
         });
         testSuite.appendChild(skipped);
-      }
-      if (executionResult.status === "timedout") {
+      } else if (executionResult.status === "timedout") {
         testCase.attributes.failures = 1;
         const failure = testCase.createNode("failure", {
           message: `Execution timeout after ${executionResult.params.allocatedMs}ms"`,
           type: "timeout",
         });
         testSuite.appendChild(failure);
-      }
-      if (executionResult.status === "failed") {
+      } else if (executionResult.status === "failed") {
         const [error] = executionResult.errors;
         if (
           error &&
@@ -115,7 +119,7 @@ export const reportAsJunitXml = async (
         propertiesNode.appendChild(propertyNode);
       }
 
-      const { consoleCalls } = executionResult;
+      const { consoleCalls = [] } = executionResult;
       let stdout = "";
       let stderr = "";
       for (const consoleCall of consoleCalls) {

package/src/execution/replace_urls.js

@@ -0,0 +1,50 @@
+// `Error: yo
+// at Object.execute (http://127.0.0.1:57300/build/src/__test__/file-throw.js:9:13)
+// at doExec (http://127.0.0.1:3000/src/__test__/file-throw.js:452:38)
+// at postOrderExec (http://127.0.0.1:3000/src/__test__/file-throw.js:448:16)
+// at http://127.0.0.1:3000/src/__test__/file-throw.js:399:18`.replace(/(?:https?|ftp|file):\/\/(.*+)$/gm, (...args) => {
+//   debugger
+// })
+export const replaceUrls = (source, replace) => {
+  return source.replace(/(?:https?|ftp|file):\/\/\S+/g, (match) => {
+    let replacement = "";
+    const lastChar = match[match.length - 1];
+
+    // hotfix because our url regex sucks a bit
+    const endsWithSeparationChar = lastChar === ")" || lastChar === ":";
+    if (endsWithSeparationChar) {
+      match = match.slice(0, -1);
+    }
+
+    const lineAndColumnPattern = /:([0-9]+):([0-9]+)$/;
+    const lineAndColumMatch = match.match(lineAndColumnPattern);
+    if (lineAndColumMatch) {
+      const lineAndColumnString = lineAndColumMatch[0];
+      const lineString = lineAndColumMatch[1];
+      const columnString = lineAndColumMatch[2];
+      replacement = replace({
+        url: match.slice(0, -lineAndColumnString.length),
+        line: lineString ? parseInt(lineString) : null,
+        column: columnString ? parseInt(columnString) : null,
+      });
+    } else {
+      const linePattern = /:([0-9]+)$/;
+      const lineMatch = match.match(linePattern);
+      if (lineMatch) {
+        const lineString = lineMatch[0];
+        replacement = replace({
+          url: match.slice(0, -lineString.length),
+          line: lineString ? parseInt(lineString) : null,
+        });
+      } else {
+        replacement = replace({
+          url: match,
+        });
+      }
+    }
+    if (endsWithSeparationChar) {
+      return `${replacement}${lastChar}`;
+    }
+    return replacement;
+  });
+};