@jsenv/core 37.0.5 → 37.1.1

package/src/kitchen/url_graph/url_graph_report.js

@@ -1,4 +1,5 @@
 import { ANSI, byteAsFileSize, distributePercentages } from "@jsenv/log";
+import { GRAPH_VISITOR } from "./url_graph_visitor.js";
 
 export const createUrlGraphSummary = (
   urlGraph,
@@ -29,67 +30,69 @@ const createUrlGraphReport = (urlGraph) => {
     other: 0,
     total: 0,
   };
-  urlGraph.urlInfoMap.forEach((urlInfo) => {
-    if (urlInfo.isRoot) {
-      return;
-    }
-    // ignore:
-    // - ignored files: we don't know their content
-    // - inline files and data files: they are already taken into account in the file where they appear
-    if (urlInfo.url.startsWith("ignore:")) {
-      return;
-    }
-    if (urlInfo.isInline) {
-      return;
-    }
-    if (urlInfo.url.startsWith("data:")) {
-      return;
-    }
 
-    // file loaded via import assertion are already inside the graph
-    // their js module equivalent are ignored to avoid counting it twice
-    // in the build graph the file targeted by import assertion will likely be gone
-    // and only the js module remain (likely bundled)
-    if (
-      urlInfo.searchParams.has("as_json_module") ||
-      urlInfo.searchParams.has("as_css_module") ||
-      urlInfo.searchParams.has("as_text_module")
-    ) {
-      return;
-    }
-    const urlContentSize = Buffer.byteLength(urlInfo.content);
-    const category = determineCategory(urlInfo);
-    if (category === "sourcemap") {
-      countGroups.sourcemaps++;
-      sizeGroups.sourcemaps += urlContentSize;
-      return;
-    }
-    countGroups.total++;
-    sizeGroups.total += urlContentSize;
-    if (category === "html") {
-      countGroups.html++;
-      sizeGroups.html += urlContentSize;
-      return;
-    }
-    if (category === "css") {
-      countGroups.css++;
-      sizeGroups.css += urlContentSize;
-      return;
-    }
-    if (category === "js") {
-      countGroups.js++;
-      sizeGroups.js += urlContentSize;
-      return;
-    }
-    if (category === "json") {
-      countGroups.json++;
-      sizeGroups.json += urlContentSize;
+  GRAPH_VISITOR.forEachUrlInfoStronglyReferenced(
+    urlGraph.rootUrlInfo,
+    (urlInfo) => {
+      // ignore:
+      // - ignored files: we don't know their content
+      // - inline files and data files: they are already taken into account in the file where they appear
+      if (urlInfo.url.startsWith("ignore:")) {
+        return;
+      }
+      if (urlInfo.isInline) {
+        return;
+      }
+      if (urlInfo.url.startsWith("data:")) {
+        return;
+      }
+
+      // file loaded via import assertion are already inside the graph
+      // their js module equivalent are ignored to avoid counting it twice
+      // in the build graph the file targeted by import assertion will likely be gone
+      // and only the js module remain (likely bundled)
+      if (
+        urlInfo.searchParams.has("as_json_module") ||
+        urlInfo.searchParams.has("as_css_module") ||
+        urlInfo.searchParams.has("as_text_module")
+      ) {
+        return;
+      }
+
+      const urlContentSize = Buffer.byteLength(urlInfo.content);
+      const category = determineCategory(urlInfo);
+      if (category === "sourcemap") {
+        countGroups.sourcemaps++;
+        sizeGroups.sourcemaps += urlContentSize;
+        return;
+      }
+      countGroups.total++;
+      sizeGroups.total += urlContentSize;
+      if (category === "html") {
+        countGroups.html++;
+        sizeGroups.html += urlContentSize;
+        return;
+      }
+      if (category === "css") {
+        countGroups.css++;
+        sizeGroups.css += urlContentSize;
+        return;
+      }
+      if (category === "js") {
+        countGroups.js++;
+        sizeGroups.js += urlContentSize;
+        return;
+      }
+      if (category === "json") {
+        countGroups.json++;
+        sizeGroups.json += urlContentSize;
+        return;
+      }
+      countGroups.other++;
+      sizeGroups.other += urlContentSize;
       return;
-    }
-    countGroups.other++;
-    sizeGroups.other += urlContentSize;
-    return;
-  });
+    },
+  );
 
   const sizesToDistribute = {};
   Object.keys(sizeGroups).forEach((groupName) => {

package/src/kitchen/url_graph/url_graph_visitor.js

@@ -99,3 +99,33 @@ GRAPH_VISITOR.findDependency = (urlInfo, visitor) => {
   iterate(urlInfo);
   return found;
 };
+
+// This function will be used in "build.js"
+// by passing rootUrlInfo as first arg
+// -> this ensure we visit only urls with strong references
+// because we start from root and ignore weak ref
+// The alternative would be to iterate on urlInfoMap
+// and call urlInfo.isUsed() but that would be more expensive
+GRAPH_VISITOR.forEachUrlInfoStronglyReferenced = (initialUrlInfo, callback) => {
+  const seen = new Set();
+  seen.add(initialUrlInfo);
+  const iterateOnReferences = (urlInfo) => {
+    for (const referenceToOther of urlInfo.referenceToOthersSet) {
+      if (referenceToOther.isWeak) {
+        continue;
+      }
+      if (referenceToOther.gotInlined()) {
+        continue;
+      }
+      const referencedUrlInfo = referenceToOther.urlInfo;
+      if (seen.has(referencedUrlInfo)) {
+        continue;
+      }
+      seen.add(referencedUrlInfo);
+      callback(referencedUrlInfo);
+      iterateOnReferences(referencedUrlInfo);
+    }
+  };
+  iterateOnReferences(initialUrlInfo);
+  seen.clear();
+};

package/src/kitchen/url_graph/url_info_transformations.js

@@ -28,11 +28,6 @@ export const createUrlInfoTransformer = ({
     sourcemapsSourcesContent = true;
   }
 
-  const sourcemapsEnabled =
-    sourcemaps === "inline" ||
-    sourcemaps === "file" ||
-    sourcemaps === "programmatic";
-
   const normalizeSourcemap = (urlInfo, sourcemap) => {
     let { sources } = sourcemap;
     if (sources) {
@@ -75,15 +70,10 @@ export const createUrlInfoTransformer = ({
     urlInfo.originalContentEtag = undefined;
     urlInfo.contentAst = undefined;
     urlInfo.contentEtag = undefined;
+    urlInfo.contentLength = undefined;
     urlInfo.content = undefined;
     urlInfo.sourcemap = null;
     urlInfo.sourcemapIsWrong = null;
-    urlInfo.referenceToOthersSet.forEach((referenceToOther) => {
-      const referencedUrlInfo = referenceToOther.urlInfo;
-      if (referencedUrlInfo.isInline) {
-        referencedUrlInfo.deleteFromGraph();
-      }
-    });
   };
 
   const setContent = async (
@@ -92,6 +82,7 @@ export const createUrlInfoTransformer = ({
     {
       contentAst, // most of the time will be undefined
       contentEtag, // in practice it's always undefined
+      contentLength,
       originalContent = content,
       originalContentAst, // most of the time will be undefined
       originalContentEtag, // in practice always undefined
@@ -107,17 +98,12 @@ export const createUrlInfoTransformer = ({
 
     urlInfo.contentAst = contentAst;
     urlInfo.contentEtag = contentEtag;
+    urlInfo.contentLength = contentLength;
     urlInfo.content = content;
     defineGettersOnPropertiesDerivedFromContent(urlInfo);
 
     urlInfo.sourcemap = sourcemap;
-    if (!sourcemapsEnabled) {
-      return;
-    }
-    if (!SOURCEMAP.enabledOnContentType(urlInfo.contentType)) {
-      return;
-    }
-    if (urlInfo.generatedUrl.startsWith("data:")) {
+    if (!shouldHandleSourcemap(urlInfo)) {
       return;
     }
     // sourcemap is a special kind of reference:
@@ -180,6 +166,7 @@ export const createUrlInfoTransformer = ({
       content,
       contentAst, // undefined most of the time
       contentEtag, // in practice always undefined
+      contentLength,
       sourcemap,
       sourcemapIsWrong,
     } = transformations;
@@ -193,10 +180,11 @@ export const createUrlInfoTransformer = ({
     if (contentModified) {
       urlInfo.contentAst = contentAst;
       urlInfo.contentEtag = contentEtag;
+      urlInfo.contentLength = contentLength;
       urlInfo.content = content;
       defineGettersOnPropertiesDerivedFromContent(urlInfo);
     }
-    if (sourcemapsEnabled && sourcemap) {
+    if (sourcemap && shouldHandleSourcemap(urlInfo)) {
       const sourcemapNormalized = normalizeSourcemap(urlInfo, sourcemap);
       const finalSourcemap = composeTwoSourcemaps(
         urlInfo.sourcemap,
@@ -272,13 +260,7 @@ export const createUrlInfoTransformer = ({
   };
 
   const applySourcemapOnContent = (urlInfo) => {
-    if (!sourcemapsEnabled) {
-      return;
-    }
-    if (!urlInfo.sourcemap) {
-      return;
-    }
-    if (urlInfo.generatedUrl.startsWith("data:")) {
+    if (!urlInfo.sourcemap || !shouldHandleSourcemap(urlInfo)) {
       return;
     }
 
@@ -363,3 +345,21 @@ export const createUrlInfoTransformer = ({
     endTransformations,
   };
 };
+
+const shouldHandleSourcemap = (urlInfo) => {
+  const { sourcemaps } = urlInfo.context;
+  if (
+    sourcemaps !== "inline" &&
+    sourcemaps !== "file" &&
+    sourcemaps !== "programmatic"
+  ) {
+    return false;
+  }
+  if (urlInfo.url.startsWith("data:")) {
+    return false;
+  }
+  if (!SOURCEMAP.enabledOnContentType(urlInfo.contentType)) {
+    return false;
+  }
+  return true;
+};

package/src/plugins/autoreload/client/autoreload.js

@@ -127,7 +127,7 @@ const dequeue = async () => {
   }
 };
 
-const applyHotReload = async ({ hotInstructions }) => {
+const applyHotReload = async ({ cause, hot, hotInstructions }) => {
   await hotInstructions.reduce(
     async (previous, { type, boundary, acceptedBy }) => {
       await previous;
@@ -143,7 +143,7 @@ const applyHotReload = async ({ hotInstructions }) => {
           delete urlHotMetas[urlToFetch];
           if (urlHotMeta.disposeCallback) {
             console.groupCollapsed(
-              `[jsenv] cleanup ${boundary} (previously used in ${acceptedBy})`,
+              `[jsenv] cleanup ${boundary} (no longer referenced by ${acceptedBy})`,
             );
             console.log(`call dispose callback`);
             await urlHotMeta.disposeCallback();
@@ -154,10 +154,10 @@ const applyHotReload = async ({ hotInstructions }) => {
       }
 
       if (acceptedBy === boundary) {
-        console.groupCollapsed(`[jsenv] hot reloading ${boundary}`);
+        console.groupCollapsed(`[jsenv] hot reloading ${boundary} (${cause})`);
       } else {
         console.groupCollapsed(
-          `[jsenv] hot reloading ${acceptedBy} usage in ${boundary}`,
+          `[jsenv] hot reloading ${acceptedBy} usage in ${boundary} (${cause})`,
         );
       }
       if (type === "js_module") {
@@ -174,7 +174,7 @@ const applyHotReload = async ({ hotInstructions }) => {
           type: "dynamic_import",
           url: urlToFetch,
         };
-        const namespace = await reloadJsImport(urlToFetch);
+        const namespace = await reloadJsImport(urlToFetch, hot);
         if (urlHotMeta.acceptCallback) {
           await urlHotMeta.acceptCallback(namespace);
         }
@@ -201,11 +201,11 @@ const applyHotReload = async ({ hotInstructions }) => {
           console.log(`no dom node using ${acceptedBy}`);
         } else if (domNodesCount === 1) {
           console.log(`reloading`, domNodesUsingUrl[0].node);
-          domNodesUsingUrl[0].reload();
+          domNodesUsingUrl[0].reload(hot);
         } else {
           console.log(`reloading ${domNodesCount} nodes using ${acceptedBy}`);
           domNodesUsingUrl.forEach((domNodesUsingUrl) => {
-            domNodesUsingUrl.reload();
+            domNodesUsingUrl.reload(hot);
           });
         }
         console.groupEnd();

package/src/plugins/autoreload/client/reload.js

@@ -31,20 +31,20 @@ export const getDOMNodesUsingUrl = (urlToReload) => {
     }
     nodes.push({
       node,
-      reload: () => {
+      reload: (hot) => {
         if (node.nodeName === "SCRIPT") {
           const copy = document.createElement("script");
           Array.from(node.attributes).forEach((attribute) => {
             copy.setAttribute(attribute.nodeName, attribute.nodeValue);
           });
-          copy.src = injectQuery(node.src, { hot: Date.now() });
+          copy.src = injectQuery(node.src, { hot });
           if (node.parentNode) {
             node.parentNode.replaceChild(copy, node);
           } else {
             document.body.appendChild(copy);
           }
         } else {
-          node[attributeName] = injectQuery(attribute, { hot: Date.now() });
+          node[attributeName] = injectQuery(attribute, { hot });
         }
       },
     });
@@ -88,16 +88,19 @@ export const getDOMNodesUsingUrl = (urlToReload) => {
     visitNodeAttributeAsUrl(img, "src");
     const srcset = img.srcset;
     if (srcset) {
-      const srcCandidates = parseSrcSet(srcset);
-      srcCandidates.forEach((srcCandidate) => {
-        const url = new URL(srcCandidate.specifier, `${window.location.href}`);
-        if (shouldReloadUrl(url)) {
-          srcCandidate.specifier = injectQuery(url, { hot: Date.now() });
-        }
-      });
       nodes.push({
         node: img,
-        reload: () => {
+        reload: (hot) => {
+          const srcCandidates = parseSrcSet(srcset);
+          srcCandidates.forEach((srcCandidate) => {
+            const url = new URL(
+              srcCandidate.specifier,
+              `${window.location.href}`,
+            );
+            if (shouldReloadUrl(url)) {
+              srcCandidate.specifier = injectQuery(url, { hot });
+            }
+          });
           img.srcset = stringifySrcSet(srcCandidates);
         },
       });
@@ -117,8 +120,8 @@ export const getDOMNodesUsingUrl = (urlToReload) => {
   return nodes;
 };
 
-export const reloadJsImport = async (url) => {
-  const urlWithHotSearchParam = injectQuery(url, { hot: Date.now() });
+export const reloadJsImport = async (url, hot) => {
+  const urlWithHotSearchParam = injectQuery(url, { hot });
   const namespace = await import(urlWithHotSearchParam);
   return namespace;
 };

package/src/plugins/autoreload/jsenv_plugin_autoreload.js

@@ -3,15 +3,15 @@ import { jsenvPluginAutoreloadClient } from "./jsenv_plugin_autoreload_client.js
 import { jsenvPluginAutoreloadServer } from "./jsenv_plugin_autoreload_server.js";
 
 export const jsenvPluginAutoreload = ({
-  clientFileChangeCallbackList,
-  clientFilesPruneCallbackList,
+  clientFileChangeEventEmitter,
+  clientFileDereferencedEventEmitter,
 }) => {
   return [
     jsenvPluginHotSearchParam(),
     jsenvPluginAutoreloadClient(),
     jsenvPluginAutoreloadServer({
-      clientFileChangeCallbackList,
-      clientFilesPruneCallbackList,
+      clientFileChangeEventEmitter,
+      clientFileDereferencedEventEmitter,
     }),
   ];
 };

package/src/plugins/autoreload/jsenv_plugin_autoreload_server.js

@@ -1,8 +1,8 @@
 import { urlIsInsideOf, urlToRelativeUrl } from "@jsenv/urls";
 
 export const jsenvPluginAutoreloadServer = ({
-  clientFileChangeCallbackList,
-  clientFilesPruneCallbackList,
+  clientFileChangeEventEmitter,
+  clientFileDereferencedEventEmitter,
 }) => {
   return {
     name: "jsenv:autoreload_server",
@@ -18,29 +18,7 @@ export const jsenvPluginAutoreloadServer = ({
           }
           return url;
         };
-        const notifyFullReload = ({ cause, reason, declinedBy }) => {
-          serverEventInfo.sendServerEvent({
-            cause,
-            type: "full",
-            typeReason: reason,
-            declinedBy,
-          });
-        };
-        const notifyPartialReload = ({ cause, reason, instructions }) => {
-          serverEventInfo.sendServerEvent({
-            cause,
-            type: "hot",
-            typeReason: reason,
-            hotInstructions: instructions,
-          });
-        };
         const propagateUpdate = (firstUrlInfo) => {
-          if (!serverEventInfo.kitchen.graph.getUrlInfo(firstUrlInfo.url)) {
-            return {
-              declined: true,
-              reason: `url not in the url graph`,
-            };
-          }
           const iterate = (urlInfo, seen) => {
             if (urlInfo.data.hotAcceptSelf) {
               return {
@@ -119,86 +97,150 @@ export const jsenvPluginAutoreloadServer = ({
           const seen = [];
           return iterate(firstUrlInfo, seen);
         };
-        clientFileChangeCallbackList.push(({ url, event }) => {
-          const onUrlInfo = (urlInfo) => {
-            if (!urlInfo.isUsed()) {
-              return false;
-            }
-            const relativeUrl = formatUrlForClient(urlInfo.url);
-            const hotUpdate = propagateUpdate(urlInfo);
-            if (hotUpdate.declined) {
-              notifyFullReload({
-                cause: `${relativeUrl} ${event}`,
-                reason: hotUpdate.reason,
-                declinedBy: hotUpdate.declinedBy,
-              });
-              return true;
-            }
-            notifyPartialReload({
-              cause: `${relativeUrl} ${event}`,
-              reason: hotUpdate.reason,
-              instructions: hotUpdate.instructions,
-            });
-            return true;
-          };
 
-          const urlInfo = serverEventInfo.kitchen.graph.getUrlInfo(url);
-          if (urlInfo) {
-            if (onUrlInfo(urlInfo)) {
-              return;
+        // We are delaying the moment we tell client how to reload because:
+        //
+        // 1. clientFileDereferencedEventEmitter can emit multiple times in a row
+        // It happens when previous references are removed by stopCollecting (in "references.js")
+        // In that case we could regroup the calls but we prefer to rely on debouncing to also cover
+        // code that would remove many url in a row by other means (like reference.remove())
+        //
+        // 2. clientFileChangeEventEmitter can emit a lot of times in a short period (git checkout for instance)
+        // In that case it's better to cooldown thanks to debouncing
+        //
+        // And we want to gather all the actions to take in response to these events because
+        // we want to favor full-reload when needed and resort to partial reload afterwards
+        // it's also important to ensure the client will fetch the server in the same order
+        const delayedActionSet = new Set();
+        let timeout;
+        const delayAction = (action) => {
+          delayedActionSet.add(action);
+          clearTimeout(timeout);
+          timeout = setTimeout(handleDelayedActions);
+        };
+
+        const handleDelayedActions = () => {
+          const actionSet = new Set(delayedActionSet);
+          delayedActionSet.clear();
+          let reloadMessage = null;
+          for (const action of actionSet) {
+            if (action.type === "change") {
+              const { changedUrlInfo, event } = action;
+              if (!changedUrlInfo.isUsed()) {
+                continue;
+              }
+              const hotUpdate = propagateUpdate(changedUrlInfo);
+              const relativeUrl = formatUrlForClient(changedUrlInfo.url);
+              if (hotUpdate.declined) {
+                reloadMessage = {
+                  cause: `${relativeUrl} ${event}`,
+                  type: "full",
+                  typeReason: hotUpdate.reason,
+                  declinedBy: hotUpdate.declinedBy,
+                };
+                break;
+              }
+              const instructions = hotUpdate.instructions;
+              if (reloadMessage) {
+                reloadMessage.hotInstructions.push(...instructions);
+              } else {
+                reloadMessage = {
+                  cause: `${relativeUrl} ${event}`,
+                  type: "hot",
+                  typeReason: hotUpdate.reason,
+                  hot: changedUrlInfo.modifiedTimestamp,
+                  hotInstructions: instructions,
+                };
+              }
+              continue;
             }
-            for (const searchParamVariant of urlInfo.searchParamVariantSet) {
-              if (onUrlInfo(searchParamVariant)) {
-                return;
+
+            if (action.type === "prune") {
+              const { prunedUrlInfo, lastReferenceFromOther } = action;
+              if (lastReferenceFromOther.type === "sourcemap_comment") {
+                // Can happen when starting dev server with sourcemaps: "file"
+                // In that case, as sourcemaps are injected, the reference
+                // are lost and sourcemap is considered as pruned
+                continue;
+              }
+              const { ownerUrlInfo } = lastReferenceFromOther;
+              if (!ownerUrlInfo.isUsed()) {
+                continue;
+              }
+              const ownerHotUpdate = propagateUpdate(ownerUrlInfo);
+              const cause = `${formatUrlForClient(
+                prunedUrlInfo.url,
+              )} is no longer referenced`;
+              // now check if we can hot update the parent resource
+              // then if we can hot update all dependencies
+              if (ownerHotUpdate.declined) {
+                reloadMessage = {
+                  cause,
+                  type: "full",
+                  typeReason: ownerHotUpdate.reason,
+                  declinedBy: ownerHotUpdate.declinedBy,
+                };
+                break;
+              }
+              // parent can hot update
+              // but pruned url info declines
+              if (prunedUrlInfo.data.hotDecline) {
+                reloadMessage = {
+                  cause,
+                  type: "full",
+                  typeReason: `a pruned file declines hot reload`,
+                  declinedBy: formatUrlForClient(prunedUrlInfo.url),
+                };
+                break;
+              }
+              const pruneInstruction = {
+                type: "prune",
+                boundary: formatUrlForClient(prunedUrlInfo.url),
+                acceptedBy: formatUrlForClient(
+                  lastReferenceFromOther.ownerUrlInfo.url,
+                ),
+              };
+              if (reloadMessage) {
+                reloadMessage.hotInstructions.push(pruneInstruction);
+              } else {
+                reloadMessage = {
+                  cause,
+                  type: "hot",
+                  typeReason: ownerHotUpdate.reason,
+                  hot: prunedUrlInfo.prunedTimestamp,
+                  hotInstructions: [pruneInstruction],
+                };
               }
             }
           }
-        });
-        clientFilesPruneCallbackList.push(
-          (prunedUrlInfo, lastReferenceFromOther) => {
-            if (lastReferenceFromOther.type === "sourcemap_comment") {
-              // Can happen when starting dev server with sourcemaps: "file"
-              // In that case, as sourcemaps are injected, the reference
-              // are lost and sourcemap is considered as pruned
-              return;
-            }
-            const parentHotUpdate = propagateUpdate(
-              lastReferenceFromOther.ownerUrlInfo,
-            );
-            const cause = `following file is no longer referenced: ${formatUrlForClient(
-              prunedUrlInfo.url,
-            )}`;
-            // now check if we can hot update the parent resource
-            // then if we can hot update all dependencies
-            if (parentHotUpdate.declined) {
-              notifyFullReload({
-                cause,
-                reason: parentHotUpdate.reason,
-                declinedBy: parentHotUpdate.declinedBy,
-              });
-              return;
-            }
-            // parent can hot update
-            const instructions = [];
-            if (prunedUrlInfo.data.hotDecline) {
-              notifyFullReload({
-                cause,
-                reason: `a pruned file declines hot reload`,
-                declinedBy: formatUrlForClient(prunedUrlInfo.url),
+          if (reloadMessage) {
+            serverEventInfo.sendServerEvent(reloadMessage);
+          }
+        };
+
+        clientFileChangeEventEmitter.on(({ url, event }) => {
+          const changedUrlInfo = serverEventInfo.kitchen.graph.getUrlInfo(url);
+          if (changedUrlInfo) {
+            delayAction({
+              type: "change",
+              changedUrlInfo,
+              event,
+            });
+            for (const searchParamVariant of changedUrlInfo.searchParamVariantSet) {
+              delayAction({
+                type: "change",
+                changedUrlInfo: searchParamVariant,
+                event,
               });
-              return;
             }
-            instructions.push({
+          }
+        });
+        clientFileDereferencedEventEmitter.on(
+          (prunedUrlInfo, lastReferenceFromOther) => {
+            delayAction({
               type: "prune",
-              boundary: formatUrlForClient(prunedUrlInfo.url),
-              acceptedBy: formatUrlForClient(
-                lastReferenceFromOther.ownerUrlInfo.url,
-              ),
-            });
-            notifyPartialReload({
-              cause,
-              reason: parentHotUpdate.reason,
-              instructions,
+              prunedUrlInfo,
+              lastReferenceFromOther,
             });
           },
         );