@jsenv/core 41.5.20 → 41.5.22

package/src/helpers/watch_dependencies.js

@@ -1,26 +1,47 @@
 /*
- * Detects when "npm install" makes a missing or outdated dependency match what
- * the project package.json declares, so the browser can be reloaded at that
- * moment.
+ * Detects the moment "npm install" changes what the running pages get from
+ * node_modules, so the browser can be reloaded at that moment:
+ *
+ * - a declared dependency stops being missing or outdated: what package.json
+ *   asks for is there;
+ * - a package the pages were served changes version: the "?v=" baked into the
+ *   urls they evaluated names a version node_modules no longer holds. Left
+ *   alone, the next hot reload of a file importing that package resolves it to
+ *   the new version and the page evaluates a second copy of the package next
+ *   to the first, two module scopes for something meant to exist once.
  *
  * node_modules is deliberately not watched: it is far too big, and an install
  * rewrites, dedupes and moves package directories around, so a watcher placed
- * on one of them is unreliable. Instead the few packages known to be missing or
- * outdated are polled, which costs a couple of readFileSync and stops as soon as
- * they are all installed. The project package.json is watched though: it is a
- * single file, and editing it is what puts a dependency out of date in the first
- * place.
+ * on one of them is unreliable. Instead a few package.json files are polled,
+ * at the cost of a stat each: the packages known to be missing or outdated,
+ * and the packages the pages were served, whose package.json the url graph
+ * keeps (that is where "?v=" comes from). The project package.json is watched
+ * though: it is a single file, and editing it is what puts a dependency out of
+ * date in the first place.
+ *
+ * Both detections run in the same tick so that an install fixing an outdated
+ * package the page runs is reported once: two reports would be two reloads,
+ * the second one landing on the page that just came back.
  */
 
 import { registerFileLifecycle } from "@jsenv/filesystem";
 
-import { readDependencyStatuses } from "../kitchen/package_dependencies.js";
+import {
+  packageNameFromSpecifier,
+  readDependencyStatuses,
+} from "../kitchen/package_dependencies.js";
 
 const POLL_INTERVAL = 500;
 
 export const watchDependencies = (
   packageDirectory,
-  { onProblem, onInstalled, onChange, pollInterval = POLL_INTERVAL },
+  {
+    getKitchens = () => [],
+    onProblem,
+    onInstalled,
+    onChange,
+    pollInterval = POLL_INTERVAL,
+  },
 ) => {
   let problemMap = new Map();
   // every path given to the browser is relative to the package directory, the
@@ -35,11 +56,49 @@ export const watchDependencies = (
   if (!packageDirectory.url) {
     return watcher;
   }
-  let timer = null;
 
-  const check = () => {
+  const checkServedVersions = (installMap) => {
+    for (const kitchen of getKitchens()) {
+      for (const urlInfo of kitchen.graph.urlInfoMap.values()) {
+        const served = readServedVersion(urlInfo);
+        if (!served) {
+          continue;
+        }
+        const installedPackageJson = readInstalledPackageJson(
+          packageDirectory,
+          served.directoryUrl,
+        );
+        if (!installedPackageJson) {
+          // the package is being rewritten, what it becomes is known once it is back
+          continue;
+        }
+        const installedVersion = installedPackageJson.version;
+        if (installedVersion === served.version) {
+          continue;
+        }
+        // the graph learns the file moved, as it would from a watcher: the
+        // files importing this package are cooked again with the new "?v="
+        // when the page comes back
+        urlInfo.onModified();
+        if (installMap.has(served.packageName)) {
+          continue;
+        }
+        installMap.set(served.packageName, {
+          packageName: served.packageName,
+          installedVersion,
+          servedVersion: served.version,
+          declaredVersion: null,
+          severity: "warning",
+        });
+      }
+    }
+  };
+
+  const checkDeclared = (installMap) => {
+    const statusMap = new Map();
     const nextProblemMap = new Map();
     for (const status of readDependencyStatuses(packageDirectory)) {
+      statusMap.set(status.packageName, status);
       if (status.state === "missing" || status.state === "outdated") {
         nextProblemMap.set(status.packageName, status);
       }
@@ -55,9 +114,21 @@ export const watchDependencies = (
       }
     }
     for (const [packageName, previousStatus] of problemMap) {
-      if (!nextProblemMap.has(packageName)) {
-        onInstalled(previousStatus);
+      if (nextProblemMap.has(packageName)) {
+        continue;
       }
+      const install = installMap.get(packageName);
+      if (install) {
+        install.declaredVersion = previousStatus.declaredVersion;
+        continue;
+      }
+      installMap.set(packageName, {
+        packageName,
+        installedVersion: statusMap.get(packageName).installedVersion,
+        servedVersion: null,
+        declaredVersion: previousStatus.declaredVersion,
+        severity: previousStatus.severity,
+      });
     }
     const changed =
       nextProblemMap.size !== problemMap.size ||
@@ -75,41 +146,92 @@ export const watchDependencies = (
     if (changed) {
       onChange(watcher.getProblems());
     }
-    if (problemMap.size === 0) {
-      stopPolling();
-    } else {
-      startPolling();
-    }
   };
 
-  const startPolling = () => {
-    if (timer) {
-      return;
+  // The declared dependencies are compared with node_modules only when
+  // something can have changed for them: the project package.json was edited,
+  // or a problem is known and an install may be fixing it. Comparing them at
+  // every tick would catch installs halfway and report a package as missing
+  // while it is being rewritten.
+  const check = ({ declared }) => {
+    const installMap = new Map();
+    checkServedVersions(installMap);
+    if (declared || problemMap.size > 0) {
+      checkDeclared(installMap);
     }
-    timer = setInterval(check, pollInterval);
-    timer.unref();
-  };
-  const stopPolling = () => {
-    if (!timer) {
-      return;
+    for (const install of installMap.values()) {
+      onInstalled(install);
     }
-    clearInterval(timer);
-    timer = null;
   };
 
   const unwatchPackageJson = registerFileLifecycle(
     new URL("package.json", packageDirectory.url),
     {
-      added: check,
-      updated: check,
+      added: () => check({ declared: true }),
+      updated: () => check({ declared: true }),
       keepProcessAlive: false,
     },
   );
-  check();
+  check({ declared: true });
+  const timer = setInterval(() => {
+    check({ declared: false });
+  }, pollInterval);
+  timer.unref();
 
   watcher.stop = () => {
-    stopPolling();
+    clearInterval(timer);
     unwatchPackageJson();
   };
   return watcher;
 };
+
+// The url graph keeps the content of every package.json it resolved a bare
+// specifier against, and that content holds the version the page received.
+// It is parsed once per content: the same string is seen at every tick.
+const servedVersionCache = new WeakMap();
+const readServedVersion = (urlInfo) => {
+  const { url, content } = urlInfo;
+  if (content === undefined) {
+    return null;
+  }
+  if (!url.startsWith("file:") || !url.endsWith("/package.json")) {
+    return null;
+  }
+  const nodeModulesIndex = url.lastIndexOf("/node_modules/");
+  if (nodeModulesIndex === -1) {
+    return null;
+  }
+  // dereferenced url infos stay in the graph; nothing imports this package anymore
+  if (urlInfo.referenceFromOthersSet.size === 0) {
+    return null;
+  }
+  const fromCache = servedVersionCache.get(urlInfo);
+  if (fromCache && fromCache.content === content) {
+    return fromCache;
+  }
+  let packageJson;
+  try {
+    packageJson = JSON.parse(content);
+  } catch {
+    return null;
+  }
+  const served = {
+    content,
+    directoryUrl: new URL("./", url).href,
+    packageName: packageNameFromSpecifier(
+      url.slice(nodeModulesIndex + "/node_modules/".length),
+    ),
+    version: packageJson.version,
+  };
+  servedVersionCache.set(urlInfo, served);
+  return served;
+};
+
+// an install in progress can be caught halfway, with a package.json not written yet
+const readInstalledPackageJson = (packageDirectory, directoryUrl) => {
+  try {
+    return packageDirectory.read(directoryUrl);
+  } catch {
+    return null;
+  }
+};

package/src/plugins/patches/jsenv_plugin_patches.js

@@ -0,0 +1,137 @@
+/*
+ * Text patches applied to files as they are served and built, keyed by file:
+ *
+ *   patches: {
+ *     "preact/dist/preact.mjs": [{ from: "a&&b", to: "a&&b&&c" }],
+ *   }
+ *
+ * A key is a url pattern relative to the root directory ("./main.js",
+ * "**\/*.css"), or a path inside a package ("preact/dist/preact.mjs") found by
+ * walking up from the root directory into node_modules, the way node does,
+ * so the key holds wherever the package manager hoists the package.
+ *
+ * Every `from` must occur exactly once in the file, otherwise the file fails
+ * to cook and says which patch did not apply: a dependency update that moved
+ * the patched code must be looked at, never silently unpatched.
+ */
+
+import { createMagicSource } from "@jsenv/sourcemap";
+import { URL_META } from "@jsenv/url-meta";
+import { asUrlWithoutSearch, urlToRelativeUrl } from "@jsenv/urls";
+import { existsSync } from "node:fs";
+
+export const jsenvPluginPatches = (rawPatches) => {
+  if (!rawPatches || Object.keys(rawPatches).length === 0) {
+    return [];
+  }
+  let findPatches;
+  const patchesPlugin = {
+    name: "jsenv:patches",
+    appliesDuring: "*",
+    init: (context) => {
+      const { rootDirectoryUrl } = context;
+      const patchesByPattern = {};
+      for (const key of Object.keys(rawPatches)) {
+        const patches = rawPatches[key];
+        assertPatches(patches, key);
+        patchesByPattern[resolvePatchKey(key, rootDirectoryUrl)] = patches;
+      }
+      const associations = URL_META.resolveAssociations(
+        { patches: patchesByPattern },
+        rootDirectoryUrl,
+      );
+      findPatches = (url) => {
+        const { patches } = URL_META.applyAssociations({
+          url: asUrlWithoutSearch(url),
+          associations,
+        });
+        return patches;
+      };
+    },
+    transformUrlContent: (urlInfo) => {
+      const patches = findPatches(urlInfo.url);
+      if (!patches) {
+        return null;
+      }
+      const { content } = urlInfo;
+      const magicSource = createMagicSource(content);
+      for (const { from, to } of patches) {
+        const start = content.indexOf(from);
+        const occurrenceCount =
+          start === -1 ? 0 : content.indexOf(from, start + 1) === -1 ? 1 : 2;
+        if (occurrenceCount !== 1) {
+          const fileRelativeUrl = urlToRelativeUrl(
+            urlInfo.url,
+            urlInfo.context.rootDirectoryUrl,
+          );
+          throw new Error(
+            `patch cannot apply on "${fileRelativeUrl}": ${JSON.stringify(from)} found ${occurrenceCount === 0 ? "nowhere" : "more than once"} in the file. The file may have changed since the patch was written.`,
+          );
+        }
+        magicSource.replace({
+          start,
+          end: start + from.length,
+          replacement: to,
+        });
+      }
+      return magicSource.toContentAndSourcemap();
+    },
+  };
+  return [patchesPlugin];
+};
+
+const assertPatches = (patches, key) => {
+  if (!Array.isArray(patches)) {
+    throw new TypeError(
+      `patches["${key}"] must be an array of { from, to }, got ${patches}`,
+    );
+  }
+  for (const patch of patches) {
+    if (
+      !patch ||
+      typeof patch.from !== "string" ||
+      patch.from === "" ||
+      typeof patch.to !== "string"
+    ) {
+      throw new TypeError(
+        `patches["${key}"] entries must be { from: string, to: string } with a non-empty "from"`,
+      );
+    }
+  }
+};
+
+// "./x", "../x", "/x", "file:///x" and "**/x" are url patterns; anything
+// else names a path inside a package, looked up in node_modules
+const resolvePatchKey = (key, rootDirectoryUrl) => {
+  if (
+    key.startsWith("./") ||
+    key.startsWith("../") ||
+    key.startsWith("/") ||
+    key.startsWith("file:") ||
+    key.startsWith("*")
+  ) {
+    return key;
+  }
+  const segments = key.split("/");
+  const packageName = key.startsWith("@")
+    ? `${segments[0]}/${segments[1]}`
+    : segments[0];
+  const pathInsidePackage = key.slice(packageName.length);
+  let directoryUrl = new URL(rootDirectoryUrl);
+  while (true) {
+    const packageDirectoryUrl = new URL(
+      `./node_modules/${packageName}/`,
+      directoryUrl,
+    );
+    if (existsSync(packageDirectoryUrl)) {
+      return String(new URL(`.${pathInsidePackage}`, packageDirectoryUrl));
+    }
+    const parentDirectoryUrl = new URL("../", directoryUrl);
+    if (parentDirectoryUrl.href === directoryUrl.href) {
+      throw new Error(
+        `patches["${key}"]: package "${packageName}" not found in any node_modules above ${rootDirectoryUrl}`,
+      );
+    }
+    directoryUrl = parentDirectoryUrl;
+  }
+};

package/src/plugins/plugins.js

@@ -11,6 +11,7 @@ import { jsenvPluginProtocolFile } from "./protocol_file/jsenv_plugin_protocol_f
 import { jsenvPluginProtocolHttp } from "./protocol_http/jsenv_plugin_protocol_http.js";
 import { jsenvPluginDirectoryReferenceEffect } from "./directory_reference_effect/jsenv_plugin_directory_reference_effect.js";
 import { jsenvPluginInjections } from "./injections/jsenv_plugin_injections.js";
+import { jsenvPluginPatches } from "./patches/jsenv_plugin_patches.js";
 import { jsenvPluginInlining } from "./inlining/jsenv_plugin_inlining.js";
 import { jsenvPluginCommonJsGlobals } from "./commonjs_globals/jsenv_plugin_commonjs_globals.js";
 import { jsenvPluginImportMetaScenarios } from "./import_meta_scenarios/jsenv_plugin_import_meta_scenarios.js";
@@ -51,6 +52,7 @@ export const getCorePlugins = ({
   directoryListing = true,
   directoryReferenceEffect,
   supervisor,
+  patches,
   injections,
   transpilation = true,
   inlining = true,
@@ -92,6 +94,9 @@ export const getCorePlugins = ({
     ...(packageBundle
       ? [jsenvPluginWorkspaceBundle({ packageDirectory })]
       : []),
+    // before everything else: what the other plugins read must be the
+    // patched file
+    ...jsenvPluginPatches(patches),
     // before reference analysis: an url written by an injection must hold its
     // final value when references are analyzed
     jsenvPluginInjections(injections),

package/src/plugins/server_events/client/server_events_client.js

@@ -4,7 +4,7 @@ events_manager: {
     const callbacksMap = new Map();
     let cleanup;
     const addCallbacks = (namedCallbacks) => {
-      let callbacksMapSize = callbacksMap.size;
+      const sizeBeforeAdd = callbacksMap.size;
       Object.keys(namedCallbacks).forEach((eventName) => {
         const callback = namedCallbacks[eventName];
         const existingCallbacks = callbacksMap.get(eventName);
@@ -17,7 +17,10 @@ events_manager: {
         }
         callbacks.push(callback);
       });
-      if (effect && callbacksMapSize === 0 && callbacksMapSize.size > 0) {
+      // the effect owns whatever the callbacks need to exist (the connection),
+      // so it runs on the edge into "someone is listening" and its cleanup runs
+      // on the edge back out of it
+      if (sizeBeforeAdd === 0 && callbacksMap.size > 0) {
         cleanup = effect();
       }
 
@@ -25,7 +28,7 @@ events_manager: {
       return () => {
         if (removed) return;
         removed = true;
-        callbacksMapSize = callbacksMap.size;
+        const sizeBeforeRemove = callbacksMap.size;
         Object.keys(namedCallbacks).forEach((eventName) => {
           const callback = namedCallbacks[eventName];
           const callbacks = callbacksMap.get(eventName);
@@ -40,12 +43,7 @@ events_manager: {
           }
         });
         namedCallbacks = null; // allow garbage collect
-        if (
-          cleanup &&
-          typeof cleanup === "function" &&
-          callbacksMapSize > 0 &&
-          callbacksMapSize.size === 0
-        ) {
+        if (cleanup && sizeBeforeRemove > 0 && callbacksMap.size === 0) {
           cleanup();
           cleanup = null;
         }
@@ -86,9 +84,14 @@ connection_manager: {
     CLOSED: "closed",
   };
 
+  // The peer is a dev server being restarted, not a flaky network: there is no
+  // attempt count or time budget past which giving up would be correct. Attempts
+  // space out to retryAfterMax and never stop, and the moments where the server
+  // is most likely to be back already (tab visible, window focused, network
+  // back) skip the wait instead of leaving a detached page on screen.
   createConnectionManager = (
     attemptConnection,
-    { logs, retry, retryAfter, retryMaxAttempt, retryAllocatedMs },
+    { logs, retry, retryAfter, retryAfterMax },
   ) => {
     const readyState = {
       value: READY_STATES.CLOSED,
@@ -103,6 +106,7 @@ connection_manager: {
     };
 
     let _disconnect = () => {};
+    let _retryNow = () => {};
     const connect = () => {
       if (
         readyState.value === READY_STATES.CONNECTING ||
@@ -111,11 +115,11 @@ connection_manager: {
         return;
       }
 
-      let retryCount = 0;
-      let msSpent = 0;
+      let retryDelay = retryAfter;
+      let retryTimeout = null;
       const attempt = () => {
+        retryTimeout = null;
         readyState.goTo(READY_STATES.CONNECTING);
-        let timeout;
         const cancelAttempt = attemptConnection({
           onClosed: () => {
             if (!retry) {
@@ -125,24 +129,6 @@ connection_manager: {
               }
               return;
             }
-            if (retryCount > retryMaxAttempt) {
-              readyState.goTo(READY_STATES.CLOSED);
-              if (logs) {
-                console.info(
-                  `[jsenv] could not connect to server after ${retryMaxAttempt} attempt`,
-                );
-              }
-              return;
-            }
-            if (retryAllocatedMs && msSpent > retryAllocatedMs) {
-              readyState.goTo(READY_STATES.CLOSED);
-              if (logs) {
-                console.info(
-                  `[jsenv] could not connect to server in less than ${retryAllocatedMs}ms`,
-                );
-              }
-              return;
-            }
             // if closed while open -> connection lost
             // otherwise it's the attempt to connect for the first time
             // or to reconnect
@@ -153,13 +139,13 @@ connection_manager: {
                 );
               }
             }
-            retryCount++;
-            timeout = setTimeout(() => {
-              msSpent += retryAfter;
-              attempt();
-            }, retryAfter);
+            retryTimeout = setTimeout(attempt, retryDelay);
+            const nextRetryDelay = retryDelay * 2;
+            retryDelay =
+              nextRetryDelay > retryAfterMax ? retryAfterMax : nextRetryDelay;
           },
           onOpen: () => {
+            retryDelay = retryAfter;
             readyState.goTo(READY_STATES.OPEN);
             if (logs) {
               // console.info(`[jsenv] connected to server`);
@@ -168,10 +154,19 @@ connection_manager: {
         });
         _disconnect = () => {
           cancelAttempt();
-          clearTimeout(timeout);
+          clearTimeout(retryTimeout);
+          retryTimeout = null;
           readyState.goTo(READY_STATES.CLOSED);
         };
       };
+      _retryNow = () => {
+        if (retryTimeout === null) {
+          return;
+        }
+        clearTimeout(retryTimeout);
+        retryDelay = retryAfter;
+        attempt();
+      };
       attempt();
     };
 
@@ -198,6 +193,9 @@ connection_manager: {
         _disconnect();
       }
     });
+    const removeServerMightBeBackListeners = listenServerMightBeBack(() => {
+      _retryNow();
+    });
 
     return {
       readyState,
@@ -205,11 +203,31 @@ connection_manager: {
       disconnect,
       destroy: () => {
         removePageUnloadListener();
+        removeServerMightBeBackListeners();
         disconnect();
       },
     };
   };
 
+  const listenServerMightBeBack = (callback) => {
+    const removeOnlineListener = listenEvent(window, "online", callback);
+    const removeFocusListener = listenEvent(window, "focus", callback);
+    const removeVisibilityChangeListener = listenEvent(
+      document,
+      "visibilitychange",
+      () => {
+        if (document.visibilityState === "visible") {
+          callback();
+        }
+      },
+    );
+    return () => {
+      removeOnlineListener();
+      removeFocusListener();
+      removeVisibilityChangeListener();
+    };
+  };
+
   // const listenPageMightFreeze = (callback) => {
   //   const removePageHideListener = listenEvent(window, "pagehide", (pageHideEvent) => {
   //     if (pageHideEvent.persisted === true) {
@@ -275,8 +293,7 @@ connection_using_websocket: {
       useEventsToManageConnection = true,
       retry = false,
       retryAfter = 1000,
-      retryMaxAttempt = Infinity,
-      retryAllocatedMs = Infinity,
+      retryAfterMax = 5000,
     } = {},
   ) => {
     const connectionManager = createConnectionManager(
@@ -319,7 +336,7 @@ connection_using_websocket: {
           }
         };
       },
-      { logs, retry, retryAfter, retryMaxAttempt, retryAllocatedMs },
+      { logs, retry, retryAfter, retryAfterMax },
     );
     const eventsManager = createEventsManager({
       effect: () => {
@@ -443,7 +460,6 @@ const serverEventsInterface = {
     const websocketConnection = createWebSocketConnection(websocketUrl, {
       logs,
       retry: true,
-      retryAllocatedMs: 10_000,
     });
 
     const { readyState, connect, disconnect, listenEvents } =