@jsenv/core 27.0.0-alpha.63 → 27.0.0-alpha.66

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic.js

@@ -19,31 +19,42 @@ import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
 import { injectQueryParams } from "@jsenv/utils/urls/url_utils.js"
 import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
 import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+import { fetchOriginalUrlInfo } from "@jsenv/utils/graph/fetch_original_url_info.js"
 
-import { fetchOriginalUrlInfo } from "../fetch_original_url_info.js"
 import { babelPluginTransformImportMetaUrl } from "./helpers/babel_plugin_transform_import_meta_url.js"
-import { jsenvPluginScriptTypeModuleAsClassic } from "./jsenv_plugin_script_type_module_as_classic.js"
-import { jsenvPluginWorkersTypeModuleAsClassic } from "./jsenv_plugin_workers_type_module_as_classic.js"
+import { jsenvPluginAsJsClassicHtml } from "./jsenv_plugin_as_js_classic_html.js"
+import { jsenvPluginAsJsClassicWorkers } from "./jsenv_plugin_as_js_classic_workers.js"
 
 const require = createRequire(import.meta.url)
 
 export const jsenvPluginAsJsClassic = ({ systemJsInjection }) => {
-  const systemJsClientFileUrl = new URL("./client/s.js", import.meta.url).href
+  const systemJsClientFileUrl = new URL(
+    "./client/s.js?js_classic",
+    import.meta.url,
+  ).href
 
   return [
-    asJsClassic({ systemJsInjection, systemJsClientFileUrl }),
-    jsenvPluginScriptTypeModuleAsClassic({
+    jsenvPluginAsJsClassicConversion({
+      systemJsInjection,
+      systemJsClientFileUrl,
+    }),
+    jsenvPluginAsJsClassicHtml({
       systemJsInjection,
       systemJsClientFileUrl,
       generateJsClassicFilename,
     }),
-    jsenvPluginWorkersTypeModuleAsClassic({
+    jsenvPluginAsJsClassicWorkers({
       generateJsClassicFilename,
     }),
   ]
 }
 
-const asJsClassic = ({ systemJsInjection, systemJsClientFileUrl }) => {
+// propagate ?as_js_classic to referenced urls
+// and perform the conversion during fetchUrlContent
+const jsenvPluginAsJsClassicConversion = ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+}) => {
   const propagateJsClassicSearchParam = (reference, context) => {
     const parentUrlInfo = context.urlGraph.getUrlInfo(reference.parentUrl)
     if (
@@ -60,9 +71,8 @@ const asJsClassic = ({ systemJsInjection, systemJsClientFileUrl }) => {
   }
 
   return {
-    name: "jsenv:as_js_classic",
+    name: "jsenv:as_js_classic_conversion",
     appliesDuring: "*",
-    // forward ?as_js_classic to referenced urls
     redirectUrl: {
       // We want to propagate transformation of js module to js classic to:
       // - import specifier (static/dynamic import + re-export)

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic_html.js

@@ -0,0 +1,302 @@
+import {
+  getHtmlNodeAttributeByName,
+  getHtmlNodeTextNode,
+  parseHtmlString,
+  removeHtmlNodeAttributeByName,
+  assignHtmlNodeAttributes,
+  stringifyHtmlAst,
+  visitHtmlAst,
+  htmlNodePosition,
+  setHtmlNodeGeneratedText,
+  injectScriptAsEarlyAsPossible,
+  createHtmlNode,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import { generateInlineContentUrl } from "@jsenv/utils/urls/inline_content_url_generator.js"
+import { injectQueryParamsIntoSpecifier } from "@jsenv/utils/urls/url_utils.js"
+
+export const jsenvPluginAsJsClassicHtml = ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+  generateJsClassicFilename,
+}) => {
+  return {
+    name: "jsenv:as_js_classic_html",
+    appliesDuring: "*",
+    transformUrlContent: {
+      html: async (urlInfo, context) => {
+        const shouldTransformScriptTypeModule =
+          !context.isSupportedOnCurrentClients("script_type_module") ||
+          !context.isSupportedOnCurrentClients("import_dynamic")
+        const htmlAst = parseHtmlString(urlInfo.content)
+        const preloadAsScriptNodes = []
+        const modulePreloadNodes = []
+        const moduleScriptNodes = []
+        const classicScriptNodes = []
+        const visitLinkNodes = (node) => {
+          if (node.nodeName !== "link") {
+            return
+          }
+          const relAttribute = getHtmlNodeAttributeByName(node, "rel")
+          const rel = relAttribute ? relAttribute.value : undefined
+          if (rel === "modulepreload") {
+            modulePreloadNodes.push(node)
+            return
+          }
+          if (rel === "preload") {
+            const asAttribute = getHtmlNodeAttributeByName(node, "as")
+            const asValue = asAttribute ? asAttribute.value : undefined
+            if (asValue === "script") {
+              preloadAsScriptNodes.push(node)
+            }
+            return
+          }
+        }
+        const visitScriptNodes = (node) => {
+          if (node.nodeName !== "script") {
+            return
+          }
+          const typeAttribute = getHtmlNodeAttributeByName(node, "type")
+          const type = typeAttribute ? typeAttribute.value : undefined
+          if (type === "module") {
+            moduleScriptNodes.push(node)
+            return
+          }
+          if (type === undefined || type === "text/javascript") {
+            classicScriptNodes.push(node)
+            return
+          }
+        }
+        visitHtmlAst(htmlAst, (node) => {
+          visitLinkNodes(node)
+          visitScriptNodes(node)
+        })
+
+        const actions = []
+        const jsModuleUrls = []
+        const convertedUrls = []
+        const getReferenceAsJsClassic = async (
+          reference,
+          {
+            // we don't cook ressource hints
+            // because they might refer to ressource that will be modified during build
+            // It also means something else HAVE to reference that url in order to cook it
+            // so that the preload is deleted by "resync_ressource_hints.js" otherwise
+            cookIt = false,
+          } = {},
+        ) => {
+          const newReferenceProps = {
+            expectedType: "js_classic",
+            specifier: injectQueryParamsIntoSpecifier(reference.specifier, {
+              as_js_classic: "",
+            }),
+            filename: generateJsClassicFilename(reference.url),
+          }
+          const [newReference, newUrlInfo] = context.referenceUtils.update(
+            reference,
+            newReferenceProps,
+          )
+          const convertedUrl = newUrlInfo.url
+          if (!convertedUrls.includes(convertedUrl)) {
+            convertedUrls.push(convertedUrl)
+          }
+          if (cookIt) {
+            // during dev it means js modules will be cooked before server sends the HTML
+            // it's ok because:
+            // - during dev script_type_module are supported (dev use a recent browser)
+            // - even if browser is not supported it still works it's jus a bit slower
+            //   because it needs to decide if systemjs will be injected or not
+            await context.cook(newUrlInfo, { reference: newReference })
+          }
+          return [newReference, newUrlInfo]
+        }
+
+        classicScriptNodes.forEach((classicScriptNode) => {
+          const srcAttribute = getHtmlNodeAttributeByName(
+            classicScriptNode,
+            "src",
+          )
+          if (srcAttribute) {
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === srcAttribute.value &&
+                ref.type === "script_src",
+            )
+            const urlObject = new URL(reference.url)
+            if (urlObject.searchParams.has("as_js_classic")) {
+              const convertedUrl = urlObject.href
+              convertedUrls.push(convertedUrl)
+              urlObject.searchParams.delete("as_js_classic")
+              const jsModuleUrl = urlObject.href
+              jsModuleUrls.push(jsModuleUrl)
+              actions.push(async () => {
+                const urlInfo = context.urlGraph.getUrlInfo(convertedUrl)
+                await context.cook(urlInfo, { reference })
+              })
+            }
+          }
+        })
+        moduleScriptNodes.forEach((moduleScriptNode) => {
+          const srcAttribute = getHtmlNodeAttributeByName(
+            moduleScriptNode,
+            "src",
+          )
+          if (srcAttribute) {
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === srcAttribute.value &&
+                ref.type === "script_src" &&
+                ref.expectedType === "js_module",
+            )
+            jsModuleUrls.push(reference.url)
+            if (shouldTransformScriptTypeModule) {
+              actions.push(async () => {
+                const [newReference] = await getReferenceAsJsClassic(
+                  reference,
+                  {
+                    cookIt: true,
+                  },
+                )
+                removeHtmlNodeAttributeByName(moduleScriptNode, "type")
+                srcAttribute.value = newReference.generatedSpecifier
+              })
+            }
+            return
+          }
+          if (shouldTransformScriptTypeModule) {
+            const textNode = getHtmlNodeTextNode(moduleScriptNode)
+            actions.push(async () => {
+              const { line, column, lineEnd, columnEnd, isOriginal } =
+                htmlNodePosition.readNodePosition(moduleScriptNode, {
+                  preferOriginal: true,
+                })
+              let inlineScriptUrl = generateInlineContentUrl({
+                url: urlInfo.url,
+                extension: ".js",
+                line,
+                column,
+                lineEnd,
+                columnEnd,
+              })
+              const [inlineReference] = context.referenceUtils.foundInline({
+                node: moduleScriptNode,
+                type: "script_src",
+                expectedType: "js_module",
+                isOriginalPosition: isOriginal,
+                // we remove 1 to the line because imagine the following html:
+                // <script>console.log('ok')</script>
+                // -> content starts same line as <script>
+                specifierLine: line - 1,
+                specifierColumn: column,
+                specifier: inlineScriptUrl,
+                contentType: "application/javascript",
+                content: textNode.value,
+              })
+              const [, newUrlInfo] = await getReferenceAsJsClassic(
+                inlineReference,
+                { cookIt: true },
+              )
+              removeHtmlNodeAttributeByName(moduleScriptNode, "type")
+              setHtmlNodeGeneratedText(moduleScriptNode, {
+                generatedText: newUrlInfo.content,
+                generatedBy: "jsenv:as_js_classic_html",
+              })
+            })
+          }
+        })
+        if (shouldTransformScriptTypeModule) {
+          preloadAsScriptNodes.forEach((preloadAsScriptNode) => {
+            const hrefAttribute = getHtmlNodeAttributeByName(
+              preloadAsScriptNode,
+              "href",
+            )
+            const href = hrefAttribute.value
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === href &&
+                ref.type === "link_href" &&
+                ref.expectedType === undefined,
+            )
+            const expectedScriptType = jsModuleUrls.includes(reference.url)
+              ? "module"
+              : "classic"
+            if (expectedScriptType === "module") {
+              actions.push(async () => {
+                // reference modified by <script type="module"> conversion
+                let newReference
+                if (reference.next) {
+                  newReference = reference.next
+                } else {
+                  // when the url is not referenced by a <script type="module">
+                  // we assume we want to preload "classic" but it might not be the case
+                  // but it's unlikely to happen and people should use "modulepreload" in that case anyway
+                  ;[newReference] = await getReferenceAsJsClassic(reference)
+                }
+                assignHtmlNodeAttributes(preloadAsScriptNode, {
+                  href: newReference.generatedSpecifier,
+                })
+                removeHtmlNodeAttributeByName(
+                  preloadAsScriptNode,
+                  "crossorigin",
+                )
+              })
+            }
+          })
+          modulePreloadNodes.forEach((modulePreloadNode) => {
+            const hrefAttribute = getHtmlNodeAttributeByName(
+              modulePreloadNode,
+              "href",
+            )
+            const href = hrefAttribute.value
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === href &&
+                ref.type === "link_href" &&
+                ref.expectedType === "js_module",
+            )
+            actions.push(async () => {
+              let newReference
+              if (reference.next) {
+                newReference = reference.next
+              } else {
+                ;[newReference] = await getReferenceAsJsClassic(reference)
+              }
+              assignHtmlNodeAttributes(modulePreloadNode, {
+                rel: "preload",
+                as: "script",
+                href: newReference.generatedSpecifier,
+              })
+            })
+          })
+        }
+
+        if (actions.length === 0) {
+          return null
+        }
+        await Promise.all(actions.map((action) => action()))
+        if (systemJsInjection) {
+          const needsSystemJs = convertedUrls.some(
+            (convertedUrl) =>
+              context.urlGraph.getUrlInfo(convertedUrl).data.jsClassicFormat ===
+              "system",
+          )
+          if (needsSystemJs) {
+            const [systemJsReference] = context.referenceUtils.inject({
+              type: "script_src",
+              expectedType: "js_classic",
+              specifier: systemJsClientFileUrl,
+            })
+            injectScriptAsEarlyAsPossible(
+              htmlAst,
+              createHtmlNode({
+                "tagName": "script",
+                "src": systemJsReference.generatedSpecifier,
+                "injected-by": "jsenv:as_js_classic_html",
+              }),
+            )
+          }
+        }
+        return stringifyHtmlAst(htmlAst)
+      },
+    },
+  }
+}

package/src/plugins/transpilation/as_js_classic/{jsenv_plugin_workers_type_module_as_classic.js → jsenv_plugin_as_js_classic_workers.js}

@@ -1,6 +1,6 @@
 import { injectQueryParams } from "@jsenv/utils/urls/url_utils.js"
 
-export const jsenvPluginWorkersTypeModuleAsClassic = ({
+export const jsenvPluginAsJsClassicWorkers = ({
   generateJsClassicFilename,
 }) => {
   const updateReference = (reference) => {
@@ -19,7 +19,7 @@ export const jsenvPluginWorkersTypeModuleAsClassic = ({
   }
 
   return {
-    name: "jsenv:workers_type_module_as_classic",
+    name: "jsenv:as_js_classic_workers",
     appliesDuring: "*",
     redirectUrl: {
       js_url_specifier: (reference, context) => {

package/src/plugins/transpilation/babel/jsenv_plugin_babel.js

@@ -7,7 +7,10 @@ import { babelPluginNewStylesheetAsJsenvImport } from "./new_stylesheet/babel_pl
 import { babelPluginGlobalThisAsJsenvImport } from "./global_this/babel_plugin_global_this_as_jsenv_import.js"
 import { babelPluginRegeneratorRuntimeAsJsenvImport } from "./regenerator_runtime/babel_plugin_regenerator_runtime_as_jsenv_import.js"
 
-export const jsenvPluginBabel = ({ getCustomBabelPlugins } = {}) => {
+export const jsenvPluginBabel = ({
+  getCustomBabelPlugins,
+  babelHelpersAsImport = true,
+} = {}) => {
   const transformWithBabel = async (urlInfo, context) => {
     const isJsModule = urlInfo.type === "js_module"
     const isWorker = urlInfo.subtype === "worker"
@@ -55,7 +58,7 @@ export const jsenvPluginBabel = ({ getCustomBabelPlugins } = {}) => {
       Object.assign(babelPluginStructure, getCustomBabelPlugins(context))
     }
 
-    if (isJsModule) {
+    if (isJsModule && babelHelpersAsImport) {
       if (!isSupported("global_this")) {
         babelPluginStructure["global-this-as-jsenv-import"] = [
           babelPluginGlobalThisAsJsenvImport,

package/src/plugins/transpilation/import_assertions/jsenv_plugin_import_assertions.js

@@ -14,8 +14,7 @@ import { urlToFilename } from "@jsenv/filesystem"
 
 import { injectQueryParams } from "@jsenv/utils/urls/url_utils.js"
 import { JS_QUOTES } from "@jsenv/utils/string/js_quotes.js"
-
-import { fetchOriginalUrlInfo } from "../fetch_original_url_info.js"
+import { fetchOriginalUrlInfo } from "@jsenv/utils/graph/fetch_original_url_info.js"
 
 export const jsenvPluginImportAssertions = () => {
   const updateReference = (reference, searchParam) => {

package/src/plugins/transpilation/jsenv_plugin_transpilation.js

@@ -19,6 +19,7 @@ export const jsenvPluginTranspilation = ({
   jsModuleAsJsClassic = true,
   systemJsInjection = true,
   topLevelAwait = true,
+  babelHelpersAsImport = true,
   getCustomBabelPlugins,
 }) => {
   return [
@@ -28,6 +29,7 @@ export const jsenvPluginTranspilation = ({
     jsenvPluginBabel({
       topLevelAwait,
       getCustomBabelPlugins,
+      babelHelpersAsImport,
     }),
     // but the conversion from js_module to js_classic
     // we want to do it after bundling

package/src/plugins/url_analysis/html/html_urls.js

@@ -51,6 +51,9 @@ export const parseAndTransformHtmlUrls = async (urlInfo, context) => {
       })
     },
   })
+  if (actions.length === 0) {
+    return null
+  }
   await Promise.all(actions.map((action) => action()))
   return {
     content: stringifyHtmlAst(htmlAst),

package/src/plugins/url_analysis/js/js_urls.js

@@ -51,10 +51,11 @@ export const parseAndTransformJsUrls = async (urlInfo, context) => {
       typePropertyNode: jsMention.typePropertyNode,
     })
     actions.push(async () => {
+      const replacement = await referenceUtils.readGeneratedSpecifier(reference)
       magicSource.replace({
         start: jsMention.specifierStart,
         end: jsMention.specifierEnd,
-        replacement: await referenceUtils.readGeneratedSpecifier(reference),
+        replacement,
       })
       if (reference.mutation) {
         reference.mutation(magicSource)

package/src/plugins/url_analysis/jsenv_plugin_url_analysis.js

@@ -1,18 +1,111 @@
+import {
+  normalizeStructuredMetaMap,
+  urlToMeta,
+  urlToRelativeUrl,
+} from "@jsenv/filesystem"
+
 import { parseAndTransformHtmlUrls } from "./html/html_urls.js"
 import { parseAndTransformCssUrls } from "./css/css_urls.js"
 import { parseAndTransformJsUrls } from "./js/js_urls.js"
 import { parseAndTransformWebmanifestUrls } from "./webmanifest/webmanifest_urls.js"
 
-export const jsenvPluginUrlAnalysis = () => {
+export const jsenvPluginUrlAnalysis = ({ rootDirectoryUrl, include }) => {
+  let getIncludeInfo = () => undefined
+  if (include) {
+    const includeMetaMap = normalizeStructuredMetaMap(
+      {
+        include,
+      },
+      rootDirectoryUrl,
+    )
+    getIncludeInfo = (url) => {
+      const meta = urlToMeta({
+        url,
+        structuredMetaMap: includeMetaMap,
+      })
+      return meta.include
+    }
+  }
+
   return {
     name: "jsenv:url_analysis",
     appliesDuring: "*",
+    redirectUrl: (reference) => {
+      if (
+        reference.specifier[0] === "#" &&
+        // For Html, css and in general "#" refer to a ressource in the page
+        // so that urls must be kept intact
+        // However for js import specifiers they have a different meaning and we want
+        // to resolve them (https://nodejs.org/api/packages.html#imports for instance)
+        reference.type !== "js_import_export"
+      ) {
+        reference.shouldHandle = false
+        return
+      }
+      const includeInfo = getIncludeInfo(reference.url)
+      if (includeInfo === true) {
+        reference.shouldHandle = true
+        return
+      }
+      if (includeInfo === false) {
+        reference.shouldHandle = false
+        return
+      }
+      if (reference.url.startsWith("data:")) {
+        reference.shouldHandle = true
+        return
+      }
+      if (reference.url.startsWith("file:")) {
+        reference.shouldHandle = true
+        return
+      }
+    },
     transformUrlContent: {
       html: parseAndTransformHtmlUrls,
       css: parseAndTransformCssUrls,
       js_classic: parseAndTransformJsUrls,
       js_module: parseAndTransformJsUrls,
       webmanifest: parseAndTransformWebmanifestUrls,
+      directory: (urlInfo, context) => {
+        const originalDirectoryReference = findOriginalDirectoryReference(
+          urlInfo,
+          context,
+        )
+        const directoryRelativeUrl = urlToRelativeUrl(
+          urlInfo.url,
+          context.rootDirectoryUrl,
+        )
+        JSON.parse(urlInfo.content).forEach((directoryEntryName) => {
+          context.referenceUtils.found({
+            type: "filesystem",
+            subtype: "directory_entry",
+            specifier: directoryEntryName,
+            trace: `"${directoryRelativeUrl}${directoryEntryName}" entry in directory referenced by ${originalDirectoryReference.trace}`,
+          })
+        })
+      },
     },
   }
 }
+
+const findOriginalDirectoryReference = (urlInfo, context) => {
+  const findNonFileSystemAncestor = (urlInfo) => {
+    for (const dependentUrl of urlInfo.dependents) {
+      const dependentUrlInfo = context.urlGraph.getUrlInfo(dependentUrl)
+      if (dependentUrlInfo.type !== "directory") {
+        return [dependentUrlInfo, urlInfo]
+      }
+      const found = findNonFileSystemAncestor(dependentUrlInfo)
+      if (found) {
+        return found
+      }
+    }
+    return []
+  }
+  const [ancestor, child] = findNonFileSystemAncestor(urlInfo)
+  if (!ancestor) {
+    return null
+  }
+  const ref = ancestor.references.find((ref) => ref.url === child.url)
+  return ref
+}

package/src/plugins/url_analysis/webmanifest/webmanifest_urls.js

@@ -12,6 +12,9 @@ export const parseAndTransformWebmanifestUrls = async (urlInfo, context) => {
       icon.src = await context.referenceUtils.readGeneratedSpecifier(reference)
     })
   })
+  if (actions.length === 0) {
+    return null
+  }
   await Promise.all(actions.map((action) => action()))
   return JSON.stringify(manifest, null, "  ")
 }

package/src/plugins/url_resolution/jsenv_plugin_url_resolution.js

@@ -1,8 +1,5 @@
 export const jsenvPluginUrlResolution = () => {
   const urlResolver = (reference) => {
-    if (reference.specifier[0] === "#") {
-      reference.external = true
-    }
     return new URL(
       reference.specifier,
       reference.baseUrl || reference.parentUrl,