@jsenv/core 36.0.1 → 36.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/core",
-  "version": "36.0.1",
+  "version": "36.1.0",
   "description": "Tool to develop, test and build js projects",
   "license": "MIT",
   "author": {
@@ -69,13 +69,13 @@
     "@jsenv/integrity": "0.0.1",
     "@jsenv/log": "3.3.5",
     "@jsenv/node-esm-resolution": "1.0.1",
+    "@jsenv/plugin-supervisor": "1.0.0",
     "@jsenv/server": "15.0.3",
     "@jsenv/sourcemap": "1.0.10",
     "@jsenv/url-meta": "8.1.0",
     "@jsenv/urls": "2.0.0",
     "@jsenv/utils": "2.0.1",
     "construct-style-sheets-polyfill": "3.1.0",
-    "launch-editor": "2.6.0",
     "lightningcss": "1.20.0"
   },
   "devDependencies": {
@@ -92,11 +92,11 @@
     "@jsenv/plugin-placeholders": "./packages/jsenv-plugin-placeholders/",
     "@jsenv/plugin-as-js-classic": "./packages/jsenv-plugin-as-js-classic/",
     "@jsenv/test": "./packages/test/",
-    "eslint": "8.40.0",
+    "eslint": "8.41.0",
     "eslint-plugin-html": "7.1.0",
     "eslint-plugin-import": "2.27.5",
     "eslint-plugin-react": "7.32.2",
-    "playwright": "1.33.0",
+    "playwright": "1.34.2",
     "prettier": "2.8.8"
   }
 }

package/src/build/build.js

@@ -130,14 +130,15 @@ export const build = async ({
   buildDirectoryUrl,
   entryPoints = {},
   assetsDirectory = "",
+  ignore,
 
   runtimeCompat = defaultRuntimeCompat,
   base = runtimeCompat.node ? "./" : "/",
   plugins = [],
   referenceAnalysis = {},
   nodeEsmResolution,
-  webResolution,
-  fileSystemMagicRedirection,
+  magicExtensions,
+  magicDirectoryIndex,
   directoryReferenceAllowed,
   scenarioPlaceholders,
   transpilation = {},
@@ -311,6 +312,10 @@ build ${entryPointKeys.length} entry points`);
       signal,
       logLevel,
       rootDirectoryUrl: sourceDirectoryUrl,
+      ignore,
+      // during first pass (craft) we keep "ignore:" when a reference is ignored
+      // so that the second pass (shape) properly ignore those urls
+      ignoreProtocol: "keep",
       urlGraph: rawGraph,
       build: true,
       runtimeCompat,
@@ -319,12 +324,6 @@ build ${entryPointKeys.length} entry points`);
         ...plugins,
         {
           appliesDuring: "build",
-          formatReference: (reference) => {
-            if (!reference.shouldHandle) {
-              return `ignore:${reference.specifier}`;
-            }
-            return null;
-          },
           fetchUrlContent: (urlInfo, context) => {
             if (context.reference.original) {
               rawRedirections.set(
@@ -338,17 +337,17 @@ build ${entryPointKeys.length} entry points`);
           rootDirectoryUrl: sourceDirectoryUrl,
           urlGraph: rawGraph,
           runtimeCompat,
-
           referenceAnalysis,
           nodeEsmResolution,
-          webResolution,
-          fileSystemMagicRedirection,
+          magicExtensions,
+          magicDirectoryIndex,
           directoryReferenceAllowed,
           transpilation: {
             ...transpilation,
             babelHelpersAsImport: !explicitJsModuleFallback,
             jsModuleFallbackOnJsClassic: false,
           },
+
           inlining: false,
           scenarioPlaceholders,
         }),
@@ -383,6 +382,14 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
     const finalGraphKitchen = createKitchen({
       logLevel,
       rootDirectoryUrl: buildDirectoryUrl,
+      // here most plugins are not there
+      // - no external plugin
+      // - no plugin putting reference.mustIgnore on https urls
+      // At this stage it's only about redirecting urls to the build directory
+      // consequently only a subset or urls are supported
+      supportedProtocols: ["file:", "data:", "virtual:", "ignore:"],
+      ignore,
+      ignoreProtocol: versioning ? "keep" : "remove",
       urlGraph: finalGraph,
       build: true,
       runtimeCompat,
@@ -578,9 +585,6 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
           },
           formatReference: (reference) => {
             if (!reference.generatedUrl.startsWith("file:")) {
-              if (!versioning && reference.generatedUrl.startsWith("ignore:")) {
-                return reference.generatedUrl.slice("ignore:".length);
-              }
               return null;
             }
             if (reference.isResourceHint) {
@@ -1059,14 +1063,11 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
           const contentVersionMap = new Map();
           const hashCallbacks = [];
           GRAPH.forEach(finalGraph, (urlInfo) => {
-            if (urlInfo.url.startsWith("data:")) {
-              return;
-            }
             if (urlInfo.type === "sourcemap") {
               return;
             }
             // ignore:
-            // - inline files:
+            // - inline files and data files:
             //   they are already taken into account in the file where they appear
             // - ignored files:
             //   we don't know their content
@@ -1078,7 +1079,10 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
             if (urlInfo.isInline) {
               return;
             }
-            if (!urlInfo.shouldHandle) {
+            if (urlInfo.url.startsWith("data:")) {
+              return;
+            }
+            if (urlInfo.url.startsWith("ignore:")) {
               return;
             }
             if (urlInfo.dependents.size === 0 && !urlInfo.isEntryPoint) {
@@ -1112,7 +1116,7 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
                   );
                   if (!dependencyContentVersion) {
                     // no content generated for this dependency
-                    // (inline, data:, sourcemap, shouldHandle is false, ...)
+                    // (inline, data:, ignore:, sourcemap, ...)
                     return null;
                   }
                   if (preferWithoutVersioning(reference)) {
@@ -1178,6 +1182,8 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
           const versioningKitchen = createKitchen({
             logLevel: logger.level,
             rootDirectoryUrl: buildDirectoryUrl,
+            ignore,
+            ignoreProtocol: "remove",
             urlGraph: finalGraph,
             build: true,
             runtimeCompat,
@@ -1227,13 +1233,13 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
                   return url;
                 },
                 formatReference: (reference) => {
-                  if (!reference.shouldHandle) {
-                    if (reference.generatedUrl.startsWith("ignore:")) {
-                      return reference.generatedUrl.slice("ignore:".length);
-                    }
+                  if (reference.url.startsWith("ignore:")) {
+                    return null;
+                  }
+                  if (reference.isInline) {
                     return null;
                   }
-                  if (reference.isInline || reference.url.startsWith("data:")) {
+                  if (reference.url.startsWith("data:")) {
                     return null;
                   }
                   if (reference.isResourceHint) {
@@ -1247,9 +1253,6 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
                   if (!canUseVersionedUrl(referencedUrlInfo)) {
                     return reference.specifier;
                   }
-                  if (!referencedUrlInfo.shouldHandle) {
-                    return null;
-                  }
                   const versionedUrl = versionedUrlMap.get(reference.url);
                   if (!versionedUrl) {
                     // happens for sourcemap
@@ -1376,9 +1379,6 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
       }
       cleanup_jsenv_attributes_from_html: {
         GRAPH.forEach(finalGraph, (urlInfo) => {
-          if (!urlInfo.shouldHandle) {
-            return;
-          }
           if (!urlInfo.url.startsWith("file:")) {
             return;
           }
@@ -1561,10 +1561,10 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
         if (serviceWorkerEntryUrlInfos.length > 0) {
           const serviceWorkerResources = {};
           GRAPH.forEach(finalGraph, (urlInfo) => {
-            if (urlInfo.isInline || !urlInfo.shouldHandle) {
+            if (!urlInfo.url.startsWith("file:")) {
               return;
             }
-            if (!urlInfo.url.startsWith("file:")) {
+            if (urlInfo.isInline) {
               return;
             }
             if (!canUseVersionedUrl(urlInfo)) {
@@ -1634,9 +1634,6 @@ ${ANSI.color(buildUrl, ANSI.MAGENTA)}
       return buildRelativeUrl;
     };
     GRAPH.forEach(finalGraph, (urlInfo) => {
-      if (!urlInfo.shouldHandle) {
-        return;
-      }
       if (!urlInfo.url.startsWith("file:")) {
         return;
       }

package/src/dev/file_service.js

@@ -22,14 +22,15 @@ export const createFileService = ({
 
   sourceDirectoryUrl,
   sourceMainFilePath,
+  ignore,
   sourceFilesConfig,
   runtimeCompat,
 
   plugins,
   referenceAnalysis,
   nodeEsmResolution,
-  webResolution,
-  fileSystemMagicRedirection,
+  magicExtensions,
+  magicDirectoryIndex,
   supervisor,
   transpilation,
   clientAutoreload,
@@ -97,6 +98,7 @@ export const createFileService = ({
       logLevel,
       rootDirectoryUrl: sourceDirectoryUrl,
       mainFilePath: sourceMainFilePath,
+      ignore,
       urlGraph,
       dev: true,
       runtimeCompat,
@@ -116,8 +118,8 @@ export const createFileService = ({
 
           referenceAnalysis,
           nodeEsmResolution,
-          webResolution,
-          fileSystemMagicRedirection,
+          magicExtensions,
+          magicDirectoryIndex,
           supervisor,
           transpilation,
 

package/src/dev/start_dev_server.js

@@ -25,6 +25,7 @@ import { createFileService } from "./file_service.js";
 export const startDevServer = async ({
   sourceDirectoryUrl,
   sourceMainFilePath = "./index.html",
+  ignore,
   port = 3456,
   hostname,
   acceptAnyIp,
@@ -52,9 +53,9 @@ export const startDevServer = async ({
   plugins = [],
   referenceAnalysis = {},
   nodeEsmResolution,
-  webResolution,
   supervisor = true,
-  fileSystemMagicRedirection,
+  magicExtensions,
+  magicDirectoryIndex,
   transpilation,
   cacheControl = true,
   ribbon = true,
@@ -186,14 +187,15 @@ export const startDevServer = async ({
 
           sourceDirectoryUrl,
           sourceMainFilePath,
+          ignore,
           sourceFilesConfig,
           runtimeCompat,
 
           plugins,
           referenceAnalysis,
           nodeEsmResolution,
-          webResolution,
-          fileSystemMagicRedirection,
+          magicExtensions,
+          magicDirectoryIndex,
           supervisor,
           transpilation,
           clientAutoreload,

package/src/kitchen/kitchen.js

@@ -6,6 +6,7 @@ import {
   normalizeUrl,
   setUrlFilename,
 } from "@jsenv/urls";
+import { URL_META } from "@jsenv/url-meta";
 import { writeFileSync, ensureWindowsDriveLetter } from "@jsenv/filesystem";
 import { createLogger, createDetailedMessage, ANSI } from "@jsenv/log";
 import { CONTENT_TYPE } from "@jsenv/utils/src/content_type/content_type.js";
@@ -29,6 +30,9 @@ export const createKitchen = ({
 
   rootDirectoryUrl,
   mainFilePath,
+  ignore,
+  ignoreProtocol = "remove",
+  supportedProtocols = ["file:", "data:", "virtual:", "http:", "https:"],
   urlGraph,
   dev = false,
   build = false,
@@ -72,6 +76,35 @@ export const createKitchen = ({
     pluginController.pushPlugin(pluginEntry);
   });
 
+  const isIgnoredByProtocol = (url) => {
+    const { protocol } = new URL(url);
+    const protocolIsSupported = supportedProtocols.some(
+      (supportedProtocol) => protocol === supportedProtocol,
+    );
+    return !protocolIsSupported;
+  };
+  let isIgnoredByParam = () => false;
+  if (ignore) {
+    const associations = URL_META.resolveAssociations(
+      { ignore },
+      rootDirectoryUrl,
+    );
+    const cache = new Map();
+    isIgnoredByParam = (url) => {
+      const fromCache = cache.get(url);
+      if (fromCache) return fromCache;
+      const { ignore } = URL_META.applyAssociations({
+        url,
+        associations,
+      });
+      cache.set(url, ignore);
+      return ignore;
+    };
+  }
+  const isIgnored = (url) => {
+    return isIgnoredByProtocol(url) || isIgnoredByParam(url);
+  };
+
   /*
    * - "http_request"
    * - "entry_point"
@@ -116,7 +149,6 @@ export const createKitchen = ({
     specifierColumn,
     baseUrl,
     isOriginalPosition,
-    shouldHandle,
     isEntryPoint = false,
     isResourceHint = false,
     isImplicit = false,
@@ -165,7 +197,6 @@ export const createKitchen = ({
       specifierColumn,
       isOriginalPosition,
       baseUrl,
-      shouldHandle,
       isEntryPoint,
       isResourceHint,
       isImplicit,
@@ -216,13 +247,36 @@ export const createKitchen = ({
       url = normalizeUrl(url);
       let referencedUrlObject;
       let searchParams;
-      const onReferenceUrlChange = (referenceUrl) => {
+      const setReferenceUrl = (referenceUrl) => {
+        // ignored urls are prefixed with "ignore:" so that reference are associated
+        // to a dedicated urlInfo that is ignored.
+        // this way it's only once a resource is referenced by reference that is not ignored
+        // that the resource is cooked
+        if (
+          reference.specifier[0] === "#" &&
+          // For Html, css and "#" refer to a resource in the page, reference must be preserved
+          // However for js import specifiers they have a different meaning and we want
+          // to resolve them (https://nodejs.org/api/packages.html#imports for instance)
+          reference.type !== "js_import"
+        ) {
+          referenceUrl = `ignore:${referenceUrl}`;
+        } else if (isIgnored(referenceUrl)) {
+          referenceUrl = `ignore:${referenceUrl}`;
+        }
+
+        if (
+          referenceUrl.startsWith("ignore:") &&
+          !reference.specifier.startsWith("ignore:")
+        ) {
+          reference.specifier = `ignore:${reference.specifier}`;
+        }
+
         referencedUrlObject = new URL(referenceUrl);
         searchParams = referencedUrlObject.searchParams;
         reference.url = referenceUrl;
         reference.searchParams = searchParams;
       };
-      onReferenceUrlChange(url);
+      setReferenceUrl(url);
 
       if (reference.debug) {
         logger.debug(`url resolved by "${
@@ -251,7 +305,7 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
           }
           const prevReference = { ...reference };
           updateReference(prevReference, reference);
-          onReferenceUrlChange(normalizedReturnValue);
+          setReferenceUrl(normalizedReturnValue);
         },
       );
       reference.generatedUrl = reference.url;
@@ -276,13 +330,22 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
           reference.generatedUrl = normalizeUrl(referencedUrlObject.href);
         },
       );
+
       const returnValue = pluginController.callHooksUntil(
         "formatReference",
         reference,
         referenceContext,
       );
-      reference.generatedSpecifier = returnValue || reference.generatedUrl;
-      reference.generatedSpecifier = urlSpecifierEncoding.encode(reference);
+      if (reference.url.startsWith("ignore:")) {
+        if (ignoreProtocol === "remove") {
+          reference.specifier = reference.specifier.slice("ignore:".length);
+        }
+        reference.generatedSpecifier = reference.specifier;
+        reference.generatedSpecifier = urlSpecifierEncoding.encode(reference);
+      } else {
+        reference.generatedSpecifier = returnValue || reference.generatedUrl;
+        reference.generatedSpecifier = urlSpecifierEncoding.encode(reference);
+      }
       return [reference, urlInfo];
     } catch (error) {
       throw createResolveUrlError({
@@ -464,7 +527,7 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
       });
     };
 
-    if (urlInfo.shouldHandle) {
+    if (!urlInfo.url.startsWith("ignore:")) {
       // references
       const references = [];
       context.referenceUtils = {
@@ -849,11 +912,6 @@ const traceFromUrlSite = (urlSite) => {
 };
 
 const applyReferenceEffectsOnUrlInfo = (reference, urlInfo, context) => {
-  if (reference.shouldHandle) {
-    urlInfo.shouldHandle = true;
-  } else {
-    urlInfo.shouldHandle = false;
-  }
   urlInfo.originalUrl = urlInfo.originalUrl || reference.url;
 
   if (reference.isEntryPoint || isWebWorkerEntryPointReference(reference)) {

package/src/kitchen/url_graph/url_graph_report.js

@@ -30,15 +30,19 @@ const createUrlGraphReport = (urlGraph) => {
     total: 0,
   };
   urlGraph.urlInfoMap.forEach((urlInfo) => {
-    if (urlInfo.url.startsWith("data:")) {
-      return;
-    }
     // ignore:
-    // - inline files: they are already taken into account in the file where they appear
     // - ignored files: we don't know their content
-    if (urlInfo.isInline || !urlInfo.shouldHandle) {
+    // - inline files and data files: they are already taken into account in the file where they appear
+    if (urlInfo.url.startsWith("ignore:")) {
+      return;
+    }
+    if (urlInfo.isInline) {
       return;
     }
+    if (urlInfo.url.startsWith("data:")) {
+      return;
+    }
+
     // file loaded via import assertion are already inside the graph
     // their js module equivalent are ignored to avoid counting it twice
     // in the build graph the file targeted by import assertion will likely be gone

package/src/kitchen/url_graph.js

@@ -268,7 +268,6 @@ const createUrlInfo = (url) => {
     originalUrl: undefined,
     filename: "",
     isEntryPoint: false,
-    shouldHandle: undefined,
     originalContent: undefined,
     content: undefined,
 

package/src/plugins/plugin_controller.js

@@ -153,7 +153,11 @@ export const createPluginController = (kitchenContext) => {
       info.timing[`${hook.name}-${hook.plugin.name.replace("jsenv:", "")}`] =
         performance.now() - startTimestamp;
     }
-    valueReturned = assertAndNormalizeReturnValue(hook.name, valueReturned);
+    valueReturned = assertAndNormalizeReturnValue(
+      hook.name,
+      valueReturned,
+      info,
+    );
     return valueReturned;
   };
   const callAsyncHook = async (hook, info, context) => {
@@ -176,7 +180,11 @@ export const createPluginController = (kitchenContext) => {
       info.timing[`${hook.name}-${hook.plugin.name.replace("jsenv:", "")}`] =
         performance.now() - startTimestamp;
     }
-    valueReturned = assertAndNormalizeReturnValue(hook.name, valueReturned);
+    valueReturned = assertAndNormalizeReturnValue(
+      hook.name,
+      valueReturned,
+      info,
+    );
     return valueReturned;
   };
 
@@ -277,7 +285,7 @@ const getHookFunction = (
   return hookValue;
 };
 
-const assertAndNormalizeReturnValue = (hookName, returnValue) => {
+const assertAndNormalizeReturnValue = (hookName, returnValue, info) => {
   // all hooks are allowed to return null/undefined as a signal of "I don't do anything"
   if (returnValue === null || returnValue === undefined) {
     return returnValue;
@@ -286,7 +294,7 @@ const assertAndNormalizeReturnValue = (hookName, returnValue) => {
     if (!returnValueAssertion.appliesTo.includes(hookName)) {
       continue;
     }
-    const assertionResult = returnValueAssertion.assertion(returnValue);
+    const assertionResult = returnValueAssertion.assertion(returnValue, info);
     if (assertionResult !== undefined) {
       // normalization
       returnValue = assertionResult;
@@ -320,13 +328,13 @@ const returnValueAssertions = [
       "finalizeUrlContent",
       "optimizeUrlContent",
     ],
-    assertion: (valueReturned) => {
+    assertion: (valueReturned, urlInfo) => {
       if (typeof valueReturned === "string" || Buffer.isBuffer(valueReturned)) {
         return { content: valueReturned };
       }
       if (typeof valueReturned === "object") {
-        const { shouldHandle, content, body } = valueReturned;
-        if (shouldHandle === false) {
+        const { content, body } = valueReturned;
+        if (urlInfo.url.startsWith("ignore:")) {
           return undefined;
         }
         if (typeof content !== "string" && !Buffer.isBuffer(content) && !body) {

package/src/plugins/plugins.js

@@ -1,12 +1,13 @@
+import { jsenvPluginSupervisor } from "@jsenv/plugin-supervisor";
+
 import { jsenvPluginReferenceAnalysis } from "./reference_analysis/jsenv_plugin_reference_analysis.js";
 import { jsenvPluginImportmap } from "./importmap/jsenv_plugin_importmap.js";
 import { jsenvPluginNodeEsmResolution } from "./resolution_node_esm/jsenv_plugin_node_esm_resolution.js";
 import { jsenvPluginWebResolution } from "./resolution_web/jsenv_plugin_web_resolution.js";
 import { jsenvPluginVersionSearchParam } from "./version_search_param/jsenv_plugin_version_search_param.js";
-import { jsenvPluginFileUrls } from "./file_urls/jsenv_plugin_file_urls.js";
-import { jsenvPluginHttpUrls } from "./http_urls/jsenv_plugin_http_urls.js";
+import { jsenvPluginProtocolFile } from "./protocol_file/jsenv_plugin_protocol_file.js";
+import { jsenvPluginProtocolHttp } from "./protocol_http/jsenv_plugin_protocol_http.js";
 import { jsenvPluginInlining } from "./inlining/jsenv_plugin_inlining.js";
-import { jsenvPluginSupervisor } from "./supervisor/jsenv_plugin_supervisor.js";
 import { jsenvPluginCommonJsGlobals } from "./commonjs_globals/jsenv_plugin_commonjs_globals.js";
 import { jsenvPluginImportMetaScenarios } from "./import_meta_scenarios/jsenv_plugin_import_meta_scenarios.js";
 import { jsenvPluginGlobalScenarios } from "./global_scenarios/jsenv_plugin_global_scenarios.js";
@@ -25,8 +26,8 @@ export const getCorePlugins = ({
 
   referenceAnalysis = {},
   nodeEsmResolution = {},
-  webResolution = {},
-  fileSystemMagicRedirection,
+  magicExtensions,
+  magicDirectoryIndex,
   directoryReferenceAllowed,
   supervisor,
   transpilation = true,
@@ -45,9 +46,6 @@ export const getCorePlugins = ({
   if (supervisor === true) {
     supervisor = {};
   }
-  if (fileSystemMagicRedirection === true) {
-    fileSystemMagicRedirection = {};
-  }
   if (clientAutoreload === true) {
     clientAutoreload = {};
   }
@@ -68,15 +66,16 @@ export const getCorePlugins = ({
        - reference inside a js module -> resolved by node esm
        - All the rest uses web standard url resolution
      */
-    jsenvPluginFileUrls({
+    jsenvPluginProtocolFile({
       directoryReferenceAllowed,
-      ...fileSystemMagicRedirection,
+      magicExtensions,
+      magicDirectoryIndex,
     }),
-    jsenvPluginHttpUrls(),
+    jsenvPluginProtocolHttp(),
     ...(nodeEsmResolution
       ? [jsenvPluginNodeEsmResolution(nodeEsmResolution)]
       : []),
-    jsenvPluginWebResolution(webResolution),
+    jsenvPluginWebResolution(),
 
     jsenvPluginVersionSearchParam(),
     jsenvPluginCommonJsGlobals(),