@jsenv/core 27.0.0-alpha.64 → 27.0.0-alpha.65

package/src/build/build.js

@@ -26,6 +26,7 @@ import {
   asUrlUntilPathname,
   normalizeUrl,
   asUrlWithoutSearch,
+  ensurePathnameTrailingSlash,
 } from "@jsenv/utils/urls/url_utils.js"
 import { createVersionGenerator } from "@jsenv/utils/versioning/version_generator.js"
 import { generateSourcemapUrl } from "@jsenv/utils/sourcemap/sourcemap_utils.js"
@@ -35,10 +36,10 @@ import {
 } from "@jsenv/utils/html_ast/html_ast.js"
 import { sortByDependencies } from "@jsenv/utils/graph/sort_by_dependencies.js"
 
+import { createUrlGraph } from "../omega/url_graph.js"
 import { jsenvPluginUrlAnalysis } from "../plugins/url_analysis/jsenv_plugin_url_analysis.js"
 import { jsenvPluginInline } from "../plugins/inline/jsenv_plugin_inline.js"
 import { jsenvPluginAsJsClassic } from "../plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic.js"
-import { createUrlGraph } from "../omega/url_graph.js"
 import { getCorePlugins } from "../plugins/plugins.js"
 import { createKitchen } from "../omega/kitchen.js"
 import { loadUrlGraph } from "../omega/url_graph/url_graph_load.js"
@@ -87,13 +88,27 @@ export const build = async ({
   rootDirectoryUrl,
   buildDirectoryUrl,
   entryPoints = {},
+  baseUrl = "/",
 
+  // default runtimeCompat corresponds to dynamic import
+  // (meaning we can keep <script type="module">)
+  runtimeCompat = {
+    // android: "8",
+    chrome: "63",
+    edge: "79",
+    firefox: "67",
+    ios: "11.3",
+    opera: "50",
+    safari: "11.3",
+    samsung: "8.2",
+  },
   plugins = [],
   sourcemaps = false,
+  urlAnalysis = {},
   nodeEsmResolution,
   fileSystemMagicResolution,
+  directoryReferenceAllowed,
   injectedGlobals,
-  runtimeCompat,
   transpilation = {},
   bundling = true,
   minification = true,
@@ -113,7 +128,6 @@ export const build = async ({
   buildDirectoryClean = true,
   writeOnFileSystem = true,
   writeGeneratedFiles = false,
-  baseUrl = "/",
   assetManifest = true,
   assetManifestFileRelativeUrl = "asset-manifest.json",
 }) => {
@@ -161,6 +175,7 @@ build ${entryPointKeys.length} entry points`)
       disabled: infoLogsAreDisabled,
     })
     let urlCount = 0
+    const prebuildRedirections = new Map()
     const rawGraphKitchen = createKitchen({
       signal,
       logger,
@@ -180,13 +195,33 @@ build ${entryPointKeys.length} entry points`)
             prebuildTask.setRightText(urlCount)
           },
         },
+        {
+          appliesDuring: "build",
+          fetchUrlContent: (urlInfo, context) => {
+            if (context.reference.original) {
+              prebuildRedirections.set(
+                context.reference.original.url,
+                context.reference.url,
+              )
+            }
+          },
+          formatUrl: (reference) => {
+            if (!reference.shouldHandle) {
+              return `ignore:${reference.specifier}`
+            }
+            return null
+          },
+        },
         ...getCorePlugins({
           rootDirectoryUrl,
           urlGraph: rawGraph,
           scenario: "build",
+          runtimeCompat,
 
+          urlAnalysis,
           nodeEsmResolution,
           fileSystemMagicResolution,
+          directoryReferenceAllowed,
           injectedGlobals,
           transpilation: {
             ...transpilation,
@@ -229,8 +264,8 @@ build ${entryPointKeys.length} entry points`)
       buildDirectoryUrl,
     })
     const rawUrls = {}
+    const bundleRedirections = {}
     const buildUrls = {}
-    const rawUrlRedirections = {}
     const bundleUrlInfos = {}
     const bundlers = {}
     rawGraphKitchen.pluginController.plugins.forEach((plugin) => {
@@ -320,7 +355,7 @@ build ${entryPointKeys.length} entry points`)
         })
       }
     })
-    const bundleUrlRedirections = {}
+    const bundleInternalRedirections = {}
     await Object.keys(bundlers).reduce(async (previous, type) => {
       await previous
       const bundler = bundlers[type]
@@ -362,7 +397,7 @@ build ${entryPointKeys.length} entry points`)
           const buildUrl = buildUrlsGenerator.generate(url, {
             urlInfo: bundleUrlInfo,
           })
-          rawUrlRedirections[url] = buildUrl
+          bundleRedirections[url] = buildUrl
           rawUrls[buildUrl] = url
           bundleUrlInfos[buildUrl] = bundleUrlInfo
           if (buildUrl.includes("?")) {
@@ -373,7 +408,9 @@ build ${entryPointKeys.length} entry points`)
               bundlerGeneratedUrlInfo.data.bundleRelativeUrl,
               buildDirectoryUrl,
             ).href
-            bundleUrlRedirections[urlForBundler] = buildUrl
+            if (urlForBundler !== buildUrl) {
+              bundleInternalRedirections[urlForBundler] = buildUrl
+            }
           }
         })
       } catch (e) {
@@ -383,7 +420,11 @@ build ${entryPointKeys.length} entry points`)
       bundleTask.done()
     }, Promise.resolve())
 
-    const buildUrlRedirections = {}
+    const urlAnalysisPlugin = jsenvPluginUrlAnalysis({
+      rootDirectoryUrl,
+      ...urlAnalysis,
+    })
+    const postBuildRedirections = {}
     const finalGraph = createUrlGraph()
     const optimizeUrlContentHooks =
       rawGraphKitchen.pluginController.addHook("optimizeUrlContent")
@@ -397,34 +438,58 @@ build ${entryPointKeys.length} entry points`)
       runtimeCompat,
       writeGeneratedFiles,
       plugins: [
-        jsenvPluginUrlAnalysis(),
-        jsenvPluginAsJsClassic({
-          rootDirectoryUrl,
-          systemJsInjection: true,
-        }),
-        jsenvPluginInline({
-          fetchInlineUrls: false,
-        }),
+        urlAnalysisPlugin,
+        jsenvPluginAsJsClassic({ systemJsInjection: true }),
+        jsenvPluginInline({ fetchInlineUrls: false }),
         {
           name: "jsenv:postbuild",
-          appliesDuring: { build: true },
+          appliesDuring: "build",
           resolveUrl: (reference) => {
-            if (reference.specifier[0] === "#") {
-              reference.external = true
+            const performInternalRedirections = (url) => {
+              const prebuildRedirection = prebuildRedirections.get(url)
+              if (prebuildRedirection) {
+                logger.debug(
+                  `\nprebuild redirection\n${url} ->\n${prebuildRedirection}\n`,
+                )
+                url = prebuildRedirection
+              }
+              const bundleRedirection = bundleRedirections[url]
+              if (bundleRedirection) {
+                logger.debug(
+                  `\nbundler redirection\n${url} ->\n${bundleRedirection}\n`,
+                )
+                url = bundleRedirection
+              }
+              const bundleInternalRedirection = bundleInternalRedirections[url]
+              if (bundleInternalRedirection) {
+                logger.debug(
+                  `\nbundler internal redirection\n${url} ->\n${bundleInternalRedirection}\n`,
+                )
+                url = bundleInternalRedirection
+              }
+              return url
+            }
+
+            if (reference.type === "filesystem") {
+              const parentRawUrl = rawUrls[reference.parentUrl]
+              const baseUrl = ensurePathnameTrailingSlash(parentRawUrl)
+              return performInternalRedirections(
+                new URL(reference.specifier, baseUrl).href,
+              )
             }
-            const urlBeforePotentialRedirect =
-              reference.specifier[0] === "/"
-                ? new URL(reference.specifier.slice(1), buildDirectoryUrl).href
-                : new URL(reference.specifier, reference.parentUrl).href
-            const url =
-              rawUrlRedirections[urlBeforePotentialRedirect] ||
-              urlBeforePotentialRedirect
-            const urlRedirectedByBundle = bundleUrlRedirections[url]
-            if (urlRedirectedByBundle) {
-              return urlRedirectedByBundle
+            if (reference.specifier[0] === "/") {
+              return performInternalRedirections(
+                new URL(reference.specifier.slice(1), buildDirectoryUrl).href,
+              )
             }
-            return url
+            return performInternalRedirections(
+              new URL(
+                reference.specifier,
+                reference.baseUrl || reference.parentUrl,
+              ).href,
+            )
           },
+          // redirecting urls into the build directory
           redirectUrl: (reference) => {
             if (!reference.url.startsWith("file:")) {
               return null
@@ -434,11 +499,6 @@ build ${entryPointKeys.length} entry points`)
             if (rawUrl) {
               return reference.url
             }
-            // from rollup or postcss
-            const bundleUrlInfo = bundleUrlInfos[reference.url]
-            if (bundleUrlInfo) {
-              return reference.url
-            }
             // from "js_module_as_js_classic":
             //   - injecting "?as_js_classic" for the first time
             //   - injecting "?as_js_classic" because the parentUrl has it
@@ -454,9 +514,8 @@ build ${entryPointKeys.length} entry points`)
               }
               let rawUrl
               if (urlIsInsideOf(reference.url, buildDirectoryUrl)) {
-                // rawUrl = rawUrls[reference.url] || reference.url
                 const originalBuildUrl =
-                  buildUrlRedirections[referenceOriginalUrl]
+                  postBuildRedirections[referenceOriginalUrl]
                 rawUrl = originalBuildUrl
                   ? rawUrls[originalBuildUrl]
                   : reference.url
@@ -478,7 +537,7 @@ build ${entryPointKeys.length} entry points`)
                   filename: reference.filename,
                 },
               })
-              buildUrlRedirections[originalBuildUrl] = buildUrl
+              postBuildRedirections[originalBuildUrl] = buildUrl
               rawUrls[buildUrl] = rawUrl
               return buildUrl
             }
@@ -522,10 +581,12 @@ build ${entryPointKeys.length} entry points`)
               return buildUrl
             }
             const rawUrlInfo = rawGraph.getUrlInfo(reference.url)
+            const parentUrlInfo = finalGraph.getUrlInfo(reference.parentUrl)
             // files from root directory but not given to rollup nor postcss
             if (rawUrlInfo) {
               const buildUrl = buildUrlsGenerator.generate(reference.url, {
                 urlInfo: rawUrlInfo,
+                parentUrlInfo,
               })
               if (buildUrl.includes("?")) {
                 rawUrls[asUrlWithoutSearch(buildUrl)] = rawUrlInfo.url
@@ -550,6 +611,9 @@ build ${entryPointKeys.length} entry points`)
           },
           formatUrl: (reference) => {
             if (!reference.generatedUrl.startsWith("file:")) {
+              if (!versioning && reference.generatedUrl.startsWith("ignore:")) {
+                return reference.generatedUrl.slice("ignore:".length)
+              }
               return null
             }
             if (!urlIsInsideOf(reference.generatedUrl, buildDirectoryUrl)) {
@@ -557,6 +621,10 @@ build ${entryPointKeys.length} entry points`)
                 `urls should be inside build directory at this stage, found "${reference.url}"`,
               )
             }
+            if (reference.isRessourceHint) {
+              // return the raw url, we will resync at the end
+              return rawUrls[reference.url]
+            }
             // remove eventual search params and hash
             const urlUntilPathname = asUrlUntilPathname(reference.generatedUrl)
             let specifier
@@ -582,23 +650,18 @@ build ${entryPointKeys.length} entry points`)
             return specifier
           },
           fetchUrlContent: async (finalUrlInfo, context) => {
-            if (!finalUrlInfo.url.startsWith("file:")) {
-              return { external: true }
-            }
             const fromBundleOrRawGraph = (url) => {
               const bundleUrlInfo = bundleUrlInfos[url]
               if (bundleUrlInfo) {
+                logger.debug(`fetching from bundle ${url}`)
                 return bundleUrlInfo
               }
               const rawUrl = rawUrls[url] || url
               const rawUrlInfo = rawGraph.getUrlInfo(rawUrl)
               if (!rawUrlInfo) {
-                const originalBuildUrl = buildUrlRedirections[url]
-                if (originalBuildUrl) {
-                  return fromBundleOrRawGraph(originalBuildUrl)
-                }
                 throw new Error(`Cannot find url`)
               }
+              logger.debug(`fetching from raw graph ${url}`)
               if (rawUrlInfo.isInline) {
                 // Inline content, such as <script> inside html, is transformed during the previous phase.
                 // If we read the inline content it would be considered as the original content.
@@ -638,7 +701,7 @@ build ${entryPointKeys.length} entry points`)
         },
         {
           name: "jsenv:optimize",
-          appliesDuring: { build: true },
+          appliesDuring: "build",
           finalizeUrlContent: async (urlInfo, context) => {
             if (optimizeUrlContentHooks.length) {
               await rawGraphKitchen.pluginController.callAsyncHooks(
@@ -702,6 +765,7 @@ ${Object.keys(finalGraph.urlInfos).join("\n")}`,
         runtimeCompat,
         writeGeneratedFiles,
         rawGraph,
+        urlAnalysisPlugin,
         finalGraph,
         finalGraphKitchen,
         lineBreakNormalization,
@@ -709,10 +773,10 @@ ${Object.keys(finalGraph.urlInfos).join("\n")}`,
       })
     }
     GRAPH.forEach(finalGraph, (urlInfo) => {
-      if (!urlInfo.url.startsWith("file:")) {
+      if (!urlInfo.shouldHandle) {
         return
       }
-      if (urlInfo.external) {
+      if (!urlInfo.url.startsWith("file:")) {
         return
       }
       if (urlInfo.type === "html") {
@@ -734,10 +798,11 @@ ${Object.keys(finalGraph.urlInfos).join("\n")}`,
       urlInfo.data.buildUrlSpecifier = buildUrlSpecifier
     })
     await resyncRessourceHints({
+      logger,
       finalGraphKitchen,
       finalGraph,
       rawUrls,
-      buildUrls,
+      postBuildRedirections,
     })
     buildOperation.throwIfAborted()
     const cleanupActions = []
@@ -767,10 +832,13 @@ ${Object.keys(finalGraph.urlInfos).join("\n")}`,
     const buildFileContents = {}
     const buildInlineContents = {}
     GRAPH.forEach(finalGraph, (urlInfo) => {
-      if (urlInfo.external) {
+      if (!urlInfo.shouldHandle) {
+        return
+      }
+      if (!urlInfo.url.startsWith("file:")) {
         return
       }
-      if (urlInfo.url.startsWith("data:")) {
+      if (urlInfo.type === "directory") {
         return
       }
       const buildRelativeUrl = urlToRelativeUrl(
@@ -904,6 +972,7 @@ const applyUrlVersioning = async ({
   runtimeCompat,
   writeGeneratedFiles,
   rawGraph,
+  urlAnalysisPlugin,
   finalGraph,
   finalGraphKitchen,
   lineBreakNormalization,
@@ -925,7 +994,7 @@ const applyUrlVersioning = async ({
       // ignore:
       // - inline files:
       //   they are already taken into account in the file where they appear
-      // - external files
+      // - ignored files:
       //   we don't know their content
       // - unused files without reference
       //   File updated such as style.css -> style.css.js or file.js->file.es5.js
@@ -935,7 +1004,7 @@ const applyUrlVersioning = async ({
       if (urlInfo.isInline) {
         return
       }
-      if (urlInfo.external) {
+      if (!urlInfo.shouldHandle) {
         return
       }
       if (!urlInfo.data.isEntryPoint && urlInfo.dependents.size === 0) {
@@ -958,8 +1027,8 @@ const applyUrlVersioning = async ({
         lineBreakNormalization,
       })
       urlInfo.dependencies.forEach((dependencyUrl) => {
-        // this dependency is inline (data:) or remote (http://, https://)
-        if (!dependencyUrl.startsWith("file:")) {
+        // this dependency is inline
+        if (dependencyUrl.startsWith("data:")) {
           return
         }
         const dependencyUrlInfo = finalGraph.getUrlInfo(dependencyUrl)
@@ -967,7 +1036,7 @@ const applyUrlVersioning = async ({
           // this content is part of the file, no need to take into account twice
           dependencyUrlInfo.isInline ||
           // this dependency content is not known
-          dependencyUrlInfo.external
+          !dependencyUrlInfo.shouldHandle
         ) {
           return
         }
@@ -1009,7 +1078,7 @@ const applyUrlVersioning = async ({
       runtimeCompat,
       writeGeneratedFiles,
       plugins: [
-        jsenvPluginUrlAnalysis(),
+        urlAnalysisPlugin,
         jsenvPluginInline({
           fetchInlineUrls: false,
           analyzeConvertedScripts: true, // to be able to version their urls
@@ -1019,18 +1088,38 @@ const applyUrlVersioning = async ({
           name: "jsenv:versioning",
           appliesDuring: { build: true },
           resolveUrl: (reference) => {
-            if (reference.specifier[0] === "#") {
-              reference.external = true
-            }
             const buildUrl = buildUrls[reference.specifier]
             if (buildUrl) {
               return buildUrl
             }
-            const url = new URL(reference.specifier, reference.parentUrl).href
+            const urlObject = new URL(
+              reference.specifier,
+              reference.baseUrl || reference.parentUrl,
+            )
+            const url = urlObject.href
+            // during versioning we revisit the deps
+            // but the code used to enforce trailing slash on directories
+            // is not applied because "jsenv:file_url_resolution" is not used
+            // so here we search if the url with a trailing slash exists
+            if (
+              reference.type === "filesystem" &&
+              !urlObject.pathname.endsWith("/")
+            ) {
+              const urlWithTrailingSlash = `${url}/`
+              const specifier = Object.keys(buildUrls).find(
+                (key) => buildUrls[key] === urlWithTrailingSlash,
+              )
+              if (specifier) {
+                return urlWithTrailingSlash
+              }
+            }
             return url
           },
           formatUrl: (reference) => {
-            if (reference.isInline) {
+            if (reference.isInline || reference.url.startsWith("data:")) {
+              return null
+            }
+            if (reference.isRessourceHint) {
               return null
             }
             // specifier comes from "normalize" hook done a bit earlier in this file
@@ -1039,8 +1128,7 @@ const applyUrlVersioning = async ({
             if (!canUseVersionedUrl(referencedUrlInfo)) {
               return reference.specifier
             }
-            // data:* urls and so on
-            if (!referencedUrlInfo.url.startsWith("file:")) {
+            if (!referencedUrlInfo.shouldHandle) {
               return null
             }
             const versionedUrl = referencedUrlInfo.data.versionedUrl
@@ -1077,9 +1165,6 @@ const applyUrlVersioning = async ({
             return versionedSpecifier
           },
           fetchUrlContent: (versionedUrlInfo) => {
-            if (!versionedUrlInfo.url.startsWith("file:")) {
-              return { external: true }
-            }
             if (versionedUrlInfo.isInline) {
               const rawUrlInfo = rawGraph.getUrlInfo(
                 rawUrls[versionedUrlInfo.url],

package/src/build/build_urls_generator.js

@@ -1,4 +1,4 @@
-import { urlToFilename } from "@jsenv/filesystem"
+import { urlToFilename, urlToRelativeUrl } from "@jsenv/filesystem"
 
 import { memoizeByFirstArgument } from "@jsenv/utils/memoize/memoize_by_first_argument.js"
 
@@ -17,6 +17,7 @@ export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
 
   const generate = memoizeByFirstArgument((url, { urlInfo, parentUrlInfo }) => {
     const directoryPath = determineDirectoryPath({
+      buildDirectoryUrl,
       urlInfo,
       parentUrlInfo,
     })
@@ -69,9 +70,24 @@ const splitFileExtension = (filename) => {
   return [filename.slice(0, dotLastIndex), filename.slice(dotLastIndex)]
 }
 
-const determineDirectoryPath = ({ urlInfo, parentUrlInfo }) => {
+const determineDirectoryPath = ({
+  buildDirectoryUrl,
+  urlInfo,
+  parentUrlInfo,
+}) => {
+  if (urlInfo.type === "directory") {
+    return ""
+  }
+  if (parentUrlInfo && parentUrlInfo.type === "directory") {
+    const parentDirectoryPath = urlToRelativeUrl(
+      parentUrlInfo.url,
+      buildDirectoryUrl,
+    )
+    return parentDirectoryPath
+  }
   if (urlInfo.isInline) {
     const parentDirectoryPath = determineDirectoryPath({
+      buildDirectoryUrl,
       urlInfo: parentUrlInfo,
     })
     return parentDirectoryPath

package/src/build/inject_service_worker_urls.js

@@ -22,7 +22,7 @@ export const injectServiceWorkerUrls = async ({
   }
   const serviceWorkerUrls = {}
   GRAPH.forEach(finalGraph, (urlInfo) => {
-    if (urlInfo.isInline || urlInfo.external) {
+    if (urlInfo.isInline || !urlInfo.shouldHandle) {
       return
     }
     if (!urlInfo.url.startsWith("file:")) {

package/src/build/resync_ressource_hints.js

@@ -16,9 +16,11 @@ import {
 import { GRAPH } from "./graph_utils.js"
 
 export const resyncRessourceHints = async ({
+  logger,
   finalGraphKitchen,
   finalGraph,
-  buildUrls,
+  rawUrls,
+  postBuildRedirections,
 }) => {
   const ressourceHintActions = []
   GRAPH.forEach(finalGraph, (urlInfo) => {
@@ -29,20 +31,62 @@ export const resyncRessourceHints = async ({
       const htmlAst = parseHtmlString(urlInfo.content, {
         storeOriginalPositions: false,
       })
+      const actions = []
       const visitLinkWithHref = (linkNode, hrefAttribute) => {
         const href = hrefAttribute.value
         if (!href || href.startsWith("data:")) {
           return
         }
-        const buildUrl = buildUrls[href]
+        const relAttribute = getHtmlNodeAttributeByName(linkNode, "rel")
+        const rel = relAttribute ? relAttribute.value : undefined
+        const isRessourceHint = [
+          "preconnect",
+          "dns-prefetch",
+          "prefetch",
+          "preload",
+          "modulepreload",
+        ].includes(rel)
+        if (!isRessourceHint) {
+          return
+        }
+
+        let buildUrl
+        for (const key of Object.keys(rawUrls)) {
+          if (rawUrls[key] === href) {
+            buildUrl = key
+            break
+          }
+        }
+        if (!buildUrl) {
+          logger.warn(`remove ressource hint because cannot find "${href}"`)
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
+          return
+        }
+        buildUrl = postBuildRedirections[buildUrl] || buildUrl
         const urlInfo = finalGraph.getUrlInfo(buildUrl)
         if (!urlInfo) {
+          logger.warn(
+            `remove ressource hint because cannot find "${buildUrl}" in the graph`,
+          )
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
           return
         }
         if (urlInfo.dependents.size === 0) {
-          removeHtmlNode(linkNode)
+          logger.warn(
+            `remove ressource hint because "${href}" not used anymore`,
+          )
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
           return
         }
+        actions.push(() => {
+          hrefAttribute.value = urlInfo.data.buildUrlSpecifier
+        })
       }
       visitHtmlAst(htmlAst, (node) => {
         if (node.nodeName !== "link") {
@@ -54,12 +98,15 @@ export const resyncRessourceHints = async ({
         }
         visitLinkWithHref(node, hrefAttribute)
       })
-      await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
-        urlInfo,
-        {
-          content: stringifyHtmlAst(htmlAst),
-        },
-      )
+      if (actions.length) {
+        actions.forEach((action) => action())
+        await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
+          urlInfo,
+          {
+            content: stringifyHtmlAst(htmlAst),
+          },
+        )
+      }
     })
   })
   await Promise.all(

package/src/dev/plugins/explorer/client/explorer.html

@@ -4,7 +4,7 @@
     <title>Exploring</title>
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta charset="utf-8" />
-    <link rel="icon" href="FAVICON_HREF" />
+    <link rel="icon" href="./jsenv.png" />
     <style>
       button:focus,
       a:focus,