@jsenv/core 41.5.20 → 41.5.22

package/dist/start_dev_server/start_dev_server.js

@@ -236,17 +236,29 @@ const isExactVersion = (declaredVersion) => {
 };
 
 /*
- * Detects when "npm install" makes a missing or outdated dependency match what
- * the project package.json declares, so the browser can be reloaded at that
- * moment.
+ * Detects the moment "npm install" changes what the running pages get from
+ * node_modules, so the browser can be reloaded at that moment:
+ *
+ * - a declared dependency stops being missing or outdated: what package.json
+ *   asks for is there;
+ * - a package the pages were served changes version: the "?v=" baked into the
+ *   urls they evaluated names a version node_modules no longer holds. Left
+ *   alone, the next hot reload of a file importing that package resolves it to
+ *   the new version and the page evaluates a second copy of the package next
+ *   to the first, two module scopes for something meant to exist once.
  *
  * node_modules is deliberately not watched: it is far too big, and an install
  * rewrites, dedupes and moves package directories around, so a watcher placed
- * on one of them is unreliable. Instead the few packages known to be missing or
- * outdated are polled, which costs a couple of readFileSync and stops as soon as
- * they are all installed. The project package.json is watched though: it is a
- * single file, and editing it is what puts a dependency out of date in the first
- * place.
+ * on one of them is unreliable. Instead a few package.json files are polled,
+ * at the cost of a stat each: the packages known to be missing or outdated,
+ * and the packages the pages were served, whose package.json the url graph
+ * keeps (that is where "?v=" comes from). The project package.json is watched
+ * though: it is a single file, and editing it is what puts a dependency out of
+ * date in the first place.
+ *
+ * Both detections run in the same tick so that an install fixing an outdated
+ * package the page runs is reported once: two reports would be two reloads,
+ * the second one landing on the page that just came back.
  */
 
 
@@ -254,7 +266,13 @@ const POLL_INTERVAL = 500;
 
 const watchDependencies = (
   packageDirectory,
-  { onProblem, onInstalled, onChange, pollInterval = POLL_INTERVAL },
+  {
+    getKitchens = () => [],
+    onProblem,
+    onInstalled,
+    onChange,
+    pollInterval = POLL_INTERVAL,
+  },
 ) => {
   let problemMap = new Map();
   // every path given to the browser is relative to the package directory, the
@@ -269,11 +287,49 @@ const watchDependencies = (
   if (!packageDirectory.url) {
     return watcher;
   }
-  let timer = null;
 
-  const check = () => {
+  const checkServedVersions = (installMap) => {
+    for (const kitchen of getKitchens()) {
+      for (const urlInfo of kitchen.graph.urlInfoMap.values()) {
+        const served = readServedVersion(urlInfo);
+        if (!served) {
+          continue;
+        }
+        const installedPackageJson = readInstalledPackageJson(
+          packageDirectory,
+          served.directoryUrl,
+        );
+        if (!installedPackageJson) {
+          // the package is being rewritten, what it becomes is known once it is back
+          continue;
+        }
+        const installedVersion = installedPackageJson.version;
+        if (installedVersion === served.version) {
+          continue;
+        }
+        // the graph learns the file moved, as it would from a watcher: the
+        // files importing this package are cooked again with the new "?v="
+        // when the page comes back
+        urlInfo.onModified();
+        if (installMap.has(served.packageName)) {
+          continue;
+        }
+        installMap.set(served.packageName, {
+          packageName: served.packageName,
+          installedVersion,
+          servedVersion: served.version,
+          declaredVersion: null,
+          severity: "warning",
+        });
+      }
+    }
+  };
+
+  const checkDeclared = (installMap) => {
+    const statusMap = new Map();
     const nextProblemMap = new Map();
     for (const status of readDependencyStatuses(packageDirectory)) {
+      statusMap.set(status.packageName, status);
       if (status.state === "missing" || status.state === "outdated") {
         nextProblemMap.set(status.packageName, status);
       }
@@ -289,9 +345,21 @@ const watchDependencies = (
       }
     }
     for (const [packageName, previousStatus] of problemMap) {
-      if (!nextProblemMap.has(packageName)) {
-        onInstalled(previousStatus);
+      if (nextProblemMap.has(packageName)) {
+        continue;
       }
+      const install = installMap.get(packageName);
+      if (install) {
+        install.declaredVersion = previousStatus.declaredVersion;
+        continue;
+      }
+      installMap.set(packageName, {
+        packageName,
+        installedVersion: statusMap.get(packageName).installedVersion,
+        servedVersion: null,
+        declaredVersion: previousStatus.declaredVersion,
+        severity: previousStatus.severity,
+      });
     }
     const changed =
       nextProblemMap.size !== problemMap.size ||
@@ -309,45 +377,96 @@ const watchDependencies = (
     if (changed) {
       onChange(watcher.getProblems());
     }
-    if (problemMap.size === 0) {
-      stopPolling();
-    } else {
-      startPolling();
-    }
   };
 
-  const startPolling = () => {
-    if (timer) {
-      return;
+  // The declared dependencies are compared with node_modules only when
+  // something can have changed for them: the project package.json was edited,
+  // or a problem is known and an install may be fixing it. Comparing them at
+  // every tick would catch installs halfway and report a package as missing
+  // while it is being rewritten.
+  const check = ({ declared }) => {
+    const installMap = new Map();
+    checkServedVersions(installMap);
+    if (declared || problemMap.size > 0) {
+      checkDeclared(installMap);
     }
-    timer = setInterval(check, pollInterval);
-    timer.unref();
-  };
-  const stopPolling = () => {
-    if (!timer) {
-      return;
+    for (const install of installMap.values()) {
+      onInstalled(install);
     }
-    clearInterval(timer);
-    timer = null;
   };
 
   const unwatchPackageJson = registerFileLifecycle(
     new URL("package.json", packageDirectory.url),
     {
-      added: check,
-      updated: check,
+      added: () => check({ declared: true }),
+      updated: () => check({ declared: true }),
       keepProcessAlive: false,
     },
   );
-  check();
+  check({ declared: true });
+  const timer = setInterval(() => {
+    check({ declared: false });
+  }, pollInterval);
+  timer.unref();
 
   watcher.stop = () => {
-    stopPolling();
+    clearInterval(timer);
     unwatchPackageJson();
   };
   return watcher;
 };
 
+// The url graph keeps the content of every package.json it resolved a bare
+// specifier against, and that content holds the version the page received.
+// It is parsed once per content: the same string is seen at every tick.
+const servedVersionCache = new WeakMap();
+const readServedVersion = (urlInfo) => {
+  const { url, content } = urlInfo;
+  if (content === undefined) {
+    return null;
+  }
+  if (!url.startsWith("file:") || !url.endsWith("/package.json")) {
+    return null;
+  }
+  const nodeModulesIndex = url.lastIndexOf("/node_modules/");
+  if (nodeModulesIndex === -1) {
+    return null;
+  }
+  // dereferenced url infos stay in the graph; nothing imports this package anymore
+  if (urlInfo.referenceFromOthersSet.size === 0) {
+    return null;
+  }
+  const fromCache = servedVersionCache.get(urlInfo);
+  if (fromCache && fromCache.content === content) {
+    return fromCache;
+  }
+  let packageJson;
+  try {
+    packageJson = JSON.parse(content);
+  } catch {
+    return null;
+  }
+  const served = {
+    content,
+    directoryUrl: new URL("./", url).href,
+    packageName: packageNameFromSpecifier(
+      url.slice(nodeModulesIndex + "/node_modules/".length),
+    ),
+    version: packageJson.version,
+  };
+  servedVersionCache.set(urlInfo, served);
+  return served;
+};
+
+// an install in progress can be caught halfway, with a package.json not written yet
+const readInstalledPackageJson = (packageDirectory, directoryUrl) => {
+  try {
+    return packageDirectory.read(directoryUrl);
+  } catch {
+    return null;
+  }
+};
+
 const jsenvCoreDirectoryUrl = new URL("../", import.meta.url);
 
 const createPackageDirectory = ({
@@ -5652,6 +5771,140 @@ const asInheritedInjections = (injections) => {
   return inheritedInjections;
 };
 
+/*
+ * Text patches applied to files as they are served and built, keyed by file:
+ *
+ *   patches: {
+ *     "preact/dist/preact.mjs": [{ from: "a&&b", to: "a&&b&&c" }],
+ *   }
+ *
+ * A key is a url pattern relative to the root directory ("./main.js",
+ * "**\/*.css"), or a path inside a package ("preact/dist/preact.mjs") found by
+ * walking up from the root directory into node_modules, the way node does,
+ * so the key holds wherever the package manager hoists the package.
+ *
+ * Every `from` must occur exactly once in the file, otherwise the file fails
+ * to cook and says which patch did not apply: a dependency update that moved
+ * the patched code must be looked at, never silently unpatched.
+ */
+
+
+const jsenvPluginPatches = (rawPatches) => {
+  if (!rawPatches || Object.keys(rawPatches).length === 0) {
+    return [];
+  }
+  let findPatches;
+  const patchesPlugin = {
+    name: "jsenv:patches",
+    appliesDuring: "*",
+    init: (context) => {
+      const { rootDirectoryUrl } = context;
+      const patchesByPattern = {};
+      for (const key of Object.keys(rawPatches)) {
+        const patches = rawPatches[key];
+        assertPatches(patches, key);
+        patchesByPattern[resolvePatchKey(key, rootDirectoryUrl)] = patches;
+      }
+      const associations = URL_META.resolveAssociations(
+        { patches: patchesByPattern },
+        rootDirectoryUrl,
+      );
+      findPatches = (url) => {
+        const { patches } = URL_META.applyAssociations({
+          url: asUrlWithoutSearch(url),
+          associations,
+        });
+        return patches;
+      };
+    },
+    transformUrlContent: (urlInfo) => {
+      const patches = findPatches(urlInfo.url);
+      if (!patches) {
+        return null;
+      }
+      const { content } = urlInfo;
+      const magicSource = createMagicSource(content);
+      for (const { from, to } of patches) {
+        const start = content.indexOf(from);
+        const occurrenceCount =
+          start === -1 ? 0 : content.indexOf(from, start + 1) === -1 ? 1 : 2;
+        if (occurrenceCount !== 1) {
+          const fileRelativeUrl = urlToRelativeUrl(
+            urlInfo.url,
+            urlInfo.context.rootDirectoryUrl,
+          );
+          throw new Error(
+            `patch cannot apply on "${fileRelativeUrl}": ${JSON.stringify(from)} found ${occurrenceCount === 0 ? "nowhere" : "more than once"} in the file. The file may have changed since the patch was written.`,
+          );
+        }
+        magicSource.replace({
+          start,
+          end: start + from.length,
+          replacement: to,
+        });
+      }
+      return magicSource.toContentAndSourcemap();
+    },
+  };
+  return [patchesPlugin];
+};
+
+const assertPatches = (patches, key) => {
+  if (!Array.isArray(patches)) {
+    throw new TypeError(
+      `patches["${key}"] must be an array of { from, to }, got ${patches}`,
+    );
+  }
+  for (const patch of patches) {
+    if (
+      !patch ||
+      typeof patch.from !== "string" ||
+      patch.from === "" ||
+      typeof patch.to !== "string"
+    ) {
+      throw new TypeError(
+        `patches["${key}"] entries must be { from: string, to: string } with a non-empty "from"`,
+      );
+    }
+  }
+};
+
+// "./x", "../x", "/x", "file:///x" and "**/x" are url patterns; anything
+// else names a path inside a package, looked up in node_modules
+const resolvePatchKey = (key, rootDirectoryUrl) => {
+  if (
+    key.startsWith("./") ||
+    key.startsWith("../") ||
+    key.startsWith("/") ||
+    key.startsWith("file:") ||
+    key.startsWith("*")
+  ) {
+    return key;
+  }
+  const segments = key.split("/");
+  const packageName = key.startsWith("@")
+    ? `${segments[0]}/${segments[1]}`
+    : segments[0];
+  const pathInsidePackage = key.slice(packageName.length);
+  let directoryUrl = new URL(rootDirectoryUrl);
+  while (true) {
+    const packageDirectoryUrl = new URL(
+      `./node_modules/${packageName}/`,
+      directoryUrl,
+    );
+    if (existsSync(packageDirectoryUrl)) {
+      return String(new URL(`.${pathInsidePackage}`, packageDirectoryUrl));
+    }
+    const parentDirectoryUrl = new URL("../", directoryUrl);
+    if (parentDirectoryUrl.href === directoryUrl.href) {
+      throw new Error(
+        `patches["${key}"]: package "${packageName}" not found in any node_modules above ${rootDirectoryUrl}`,
+      );
+    }
+    directoryUrl = parentDirectoryUrl;
+  }
+};
+
 const jsenvPluginInliningAsDataUrl = () => {
   return {
     name: "jsenv:inlining_as_data_url",
@@ -8101,6 +8354,7 @@ const getCorePlugins = ({
   directoryListing = true,
   directoryReferenceEffect,
   supervisor,
+  patches,
   injections,
   transpilation = true,
   inlining = true,
@@ -8142,6 +8396,9 @@ const getCorePlugins = ({
     ...(packageBundle
       ? [jsenvPluginWorkspaceBundle({ packageDirectory })]
       : []),
+    // before everything else: what the other plugins read must be the
+    // patched file
+    ...jsenvPluginPatches(patches),
     // before reference analysis: an url written by an injection must hold its
     // final value when references are analyzed
     jsenvPluginInjections(injections),
@@ -12278,6 +12535,7 @@ const EXECUTED_BY_TEST_PLAN = process.argv.includes("--jsenv-test");
  * @param {boolean} [params.supervisor=true] - Script supervisor (better error reporting).
  * @param {boolean} [params.modulepreload=false] - Send `Link: <url>; rel=modulepreload` response headers listing the static import graph of a page (as far as the graph knows it). Off by default, to enable once the server runs on http/2 or http/3: over http/1.1 the preloads take the 6 connections per origin ahead of the render-blocking scripts, and pages get slower (the why, with measures, in jsenv_plugin_modulepreload.js).
  * @param {boolean|object} [params.directoryListing=true] - Directory listing pages.
+ * @param {object} [params.patches] - Text patches applied to files as they are served, as `{ file: [{ from, to }] }`. A key is a url pattern relative to sourceDirectoryUrl (`"./main.js"`) or a path inside a package (`"preact/dist/preact.mjs"`, found in node_modules the way node does). Each `from` must occur exactly once in the file, otherwise the file fails to load and says which patch did not apply.
  * @param {object} [params.injections] - Values to inject into files, as `{ urlPattern: getInjections }`. Keys are url patterns relative to sourceDirectoryUrl (`"./index.html"`, `"**\/*.js"`), values are functions receiving `urlInfo` and returning (or resolving to) an object of placeholders to replace, named `__LIKE_THIS__` by convention. In JS the value is injected as a JS literal (a string brings its own quotes), everywhere else as-is so it can be concatenated: `href="__BACKEND_URL__/users/me"`. An html url pattern also covers what is inlined in that html, so `<script>window.backendUrl = __BACKEND_URL__;</script>` shares the value with every js file of the page. See `INJECTIONS.optional` and `INJECTIONS.global`.
  * @param {object} [params.runtimeCompat] - Target runtimes; warns when dev code wouldn't survive the build.
  * @param {string} [params.sourcemaps="inline"] - Sourcemap mode.
@@ -12339,6 +12597,7 @@ const startDevServer = async ({
   magicExtensions,
   magicDirectoryIndex,
   directoryListing,
+  patches,
   injections,
   transpilation,
   cacheControl = true,
@@ -12444,6 +12703,7 @@ const startDevServer = async ({
   };
   const dependencyProblemEventEmitter = createEventEmitter();
   const dependencyWatcher = watchDependencies(packageDirectory, {
+    getKitchens: () => kitchenCache.values(),
     onChange: (problems) => {
       dependencyProblemEventEmitter.emit(problems);
     },
@@ -12465,11 +12725,30 @@ const startDevServer = async ({
         logger.info(message);
       }
     },
-    onInstalled: ({ packageName, declaredVersion, severity }) => {
-      logger.info(`"${packageName}@${declaredVersion}" is now installed`);
+    onInstalled: ({
+      packageName,
+      installedVersion,
+      servedVersion,
+      declaredVersion,
+      severity,
+    }) => {
+      if (servedVersion) {
+        logger.info(
+          `"${packageName}@${installedVersion}" is now installed, the page runs ${servedVersion}`,
+        );
+      } else {
+        logger.info(`"${packageName}@${declaredVersion}" is now installed`);
+      }
       if (severity !== "warning") {
         return;
       }
+      if (servedVersion) {
+        reloadRequestEventEmitter.emit({
+          cause: `${packageName} ${servedVersion} -> ${installedVersion} installed`,
+          reason: `a dependency the page runs changed version in node_modules`,
+        });
+        return;
+      }
       reloadRequestEventEmitter.emit({
         cause: `${packageName}@${declaredVersion} installed`,
         reason: `a dependency became available in node_modules`,
@@ -12506,6 +12785,7 @@ const startDevServer = async ({
       magicDirectoryIndex,
       directoryListing,
       supervisor,
+      patches,
       injections,
       transpilation,
       spa,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/core",
-  "version": "41.5.20",
+  "version": "41.5.22",
   "type": "module",
   "description": "Tool to develop, test and build js projects",
   "repository": {
@@ -79,7 +79,7 @@
     "@jsenv/plugin-minification": "1.7.22",
     "@jsenv/plugin-supervisor": "1.8.24",
     "@jsenv/plugin-transpilation": "1.7.9",
-    "@jsenv/server": "17.6.4",
+    "@jsenv/server": "17.6.5",
     "@jsenv/sourcemap": "1.4.9",
     "react-table": "7.8.0"
   },

package/src/build/build.js

@@ -149,6 +149,17 @@ import { jsenvPluginMappings } from "./jsenv_plugin_mappings.js";
  *          }
  *
  *        See startDevServer "ribbon" param for the full list of options.
+ * @param {object} [entryPoint.patches]
+ *        Text patches applied to files as they are built, as { file: [{ from, to }] }.
+ *        A key is a url pattern relative to sourceDirectoryUrl ("./main.js") or a path
+ *        inside a package ("preact/dist/preact.mjs", found in node_modules the way node does):
+ *
+ *          patches: {
+ *            "preact/dist/preact.mjs": [{ from: "a&&b", to: "a&&b&&c" }],
+ *          }
+ *
+ *        Each `from` must occur exactly once in the file, otherwise the build fails
+ *        and says which patch did not apply.
  * @param {object} [entryPoint.injections]
  *        Values to inject into files, as { urlPattern: getInjections }.
  *        Keys are url patterns relative to sourceDirectoryUrl ("./index.html", "**\/*.js"),
@@ -1016,6 +1027,7 @@ const entryPointDefaultParams = {
   directoryReferenceEffect: undefined,
   scenarioPlaceholders: undefined,
   ribbon: false,
+  patches: undefined,
   injections: undefined,
   transpilation: {},
   preserveComments: undefined,
@@ -1072,6 +1084,7 @@ const prepareEntryPointBuild = async (
     directoryReferenceEffect,
     scenarioPlaceholders,
     ribbon,
+    patches,
     injections,
     transpilation,
     preserveComments,
@@ -1240,6 +1253,7 @@ const prepareEntryPointBuild = async (
       magicExtensions,
       magicDirectoryIndex,
       directoryReferenceEffect,
+      patches,
       injections,
       transpilation: {
         babelHelpersAsImport: !explicitJsModuleConversion,

package/src/dev/start_dev_server.js

@@ -60,6 +60,7 @@ const EXECUTED_BY_TEST_PLAN = process.argv.includes("--jsenv-test");
  * @param {boolean} [params.supervisor=true] - Script supervisor (better error reporting).
  * @param {boolean} [params.modulepreload=false] - Send `Link: <url>; rel=modulepreload` response headers listing the static import graph of a page (as far as the graph knows it). Off by default, to enable once the server runs on http/2 or http/3: over http/1.1 the preloads take the 6 connections per origin ahead of the render-blocking scripts, and pages get slower (the why, with measures, in jsenv_plugin_modulepreload.js).
  * @param {boolean|object} [params.directoryListing=true] - Directory listing pages.
+ * @param {object} [params.patches] - Text patches applied to files as they are served, as `{ file: [{ from, to }] }`. A key is a url pattern relative to sourceDirectoryUrl (`"./main.js"`) or a path inside a package (`"preact/dist/preact.mjs"`, found in node_modules the way node does). Each `from` must occur exactly once in the file, otherwise the file fails to load and says which patch did not apply.
  * @param {object} [params.injections] - Values to inject into files, as `{ urlPattern: getInjections }`. Keys are url patterns relative to sourceDirectoryUrl (`"./index.html"`, `"**\/*.js"`), values are functions receiving `urlInfo` and returning (or resolving to) an object of placeholders to replace, named `__LIKE_THIS__` by convention. In JS the value is injected as a JS literal (a string brings its own quotes), everywhere else as-is so it can be concatenated: `href="__BACKEND_URL__/users/me"`. An html url pattern also covers what is inlined in that html, so `<script>window.backendUrl = __BACKEND_URL__;</script>` shares the value with every js file of the page. See `INJECTIONS.optional` and `INJECTIONS.global`.
  * @param {object} [params.runtimeCompat] - Target runtimes; warns when dev code wouldn't survive the build.
  * @param {string} [params.sourcemaps="inline"] - Sourcemap mode.
@@ -121,6 +122,7 @@ export const startDevServer = async ({
   magicExtensions,
   magicDirectoryIndex,
   directoryListing,
+  patches,
   injections,
   transpilation,
   cacheControl = true,
@@ -227,6 +229,7 @@ export const startDevServer = async ({
   };
   const dependencyProblemEventEmitter = createEventEmitter();
   const dependencyWatcher = watchDependencies(packageDirectory, {
+    getKitchens: () => kitchenCache.values(),
     onChange: (problems) => {
       dependencyProblemEventEmitter.emit(problems);
     },
@@ -248,11 +251,30 @@ export const startDevServer = async ({
         logger.info(message);
       }
     },
-    onInstalled: ({ packageName, declaredVersion, severity }) => {
-      logger.info(`"${packageName}@${declaredVersion}" is now installed`);
+    onInstalled: ({
+      packageName,
+      installedVersion,
+      servedVersion,
+      declaredVersion,
+      severity,
+    }) => {
+      if (servedVersion) {
+        logger.info(
+          `"${packageName}@${installedVersion}" is now installed, the page runs ${servedVersion}`,
+        );
+      } else {
+        logger.info(`"${packageName}@${declaredVersion}" is now installed`);
+      }
       if (severity !== "warning") {
         return;
       }
+      if (servedVersion) {
+        reloadRequestEventEmitter.emit({
+          cause: `${packageName} ${servedVersion} -> ${installedVersion} installed`,
+          reason: `a dependency the page runs changed version in node_modules`,
+        });
+        return;
+      }
       reloadRequestEventEmitter.emit({
         cause: `${packageName}@${declaredVersion} installed`,
         reason: `a dependency became available in node_modules`,
@@ -289,6 +311,7 @@ export const startDevServer = async ({
       magicDirectoryIndex,
       directoryListing,
       supervisor,
+      patches,
       injections,
       transpilation,
       spa,