@jsenv/core 27.0.0-alpha.7 → 27.0.0-alpha.70

package/src/plugins/transpilation/as_js_classic/helpers/systemjs_old.js

@@ -0,0 +1,43 @@
+// const { clientRuntimeCompat, isSupportedOnCurrentClient } = context
+// const shouldBeCompatibleWithNode =
+//   Object.keys(clientRuntimeCompat).includes("node")
+// const requiredFeatureNames = [
+//   "import_dynamic",
+//   "top_level_await",
+//   "global_this",
+//   // when using node we assume the code won't use browser specific feature
+//   ...(shouldBeCompatibleWithNode
+//     ? []
+//     : [
+//         "script_type_module",
+//         "worker_type_module",
+//         "import_type_json",
+//         "import_type_css",
+//       ]),
+//   // "importmap",
+// ]
+// const needsSystemJs = featuresRelatedToSystemJs.some((featureName) => {
+//   const isRequired = requiredFeatureNames.includes(featureName)
+//   return isRequired && !isSupportedOnCurrentClient(featureName)
+// })
+// if (!needsSystemJs) {
+//   return null
+// }
+// const { code, map } = await applyBabelPlugins({
+
+// })
+// return {
+//   content: code,
+//   soourcemap: map,
+// }
+
+// const featuresRelatedToSystemJs = [
+//   "script_type_module",
+//   "worker_type_module",
+//   "import_dynamic",
+//   "import_type_json",
+//   "import_type_css",
+//   "top_level_await",
+//   // "importmap",
+//   // "worker_importmap",
+// ]

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic.js

@@ -0,0 +1,213 @@
+/*
+ * Something to keep in mind:
+ * When systemjs format is used by babel, it will generated UID based on
+ * the import specifier:
+ * https://github.com/babel/babel/blob/97d1967826077f15e766778c0d64711399e9a72a/packages/babel-plugin-transform-modules-systemjs/src/index.ts#L498
+ * But at this stage import specifier are absolute file urls
+ * So without minification these specifier are long and dependent
+ * on where the files are on the filesystem.
+ * This is mitigated by minification that will shorten them
+ * But ideally babel should not generate this in the first place
+ * and prefer to unique identifier based solely on the specifier basename for instance
+ */
+
+import { createRequire } from "node:module"
+import { urlToFilename, injectQueryParams } from "@jsenv/urls"
+import { readFileSync } from "@jsenv/filesystem"
+
+import { requireBabelPlugin } from "@jsenv/babel-plugins"
+import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+
+import { babelPluginTransformImportMetaUrl } from "./helpers/babel_plugin_transform_import_meta_url.js"
+import { jsenvPluginAsJsClassicHtml } from "./jsenv_plugin_as_js_classic_html.js"
+import { jsenvPluginAsJsClassicWorkers } from "./jsenv_plugin_as_js_classic_workers.js"
+
+const require = createRequire(import.meta.url)
+
+export const jsenvPluginAsJsClassic = ({ systemJsInjection }) => {
+  const systemJsClientFileUrl = new URL(
+    "./client/s.js?js_classic",
+    import.meta.url,
+  ).href
+
+  return [
+    jsenvPluginAsJsClassicConversion({
+      systemJsInjection,
+      systemJsClientFileUrl,
+    }),
+    jsenvPluginAsJsClassicHtml({
+      systemJsInjection,
+      systemJsClientFileUrl,
+      generateJsClassicFilename,
+    }),
+    jsenvPluginAsJsClassicWorkers({
+      generateJsClassicFilename,
+    }),
+  ]
+}
+
+// propagate ?as_js_classic to referenced urls
+// and perform the conversion during fetchUrlContent
+const jsenvPluginAsJsClassicConversion = ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+}) => {
+  const propagateJsClassicSearchParam = (reference, context) => {
+    const parentUrlInfo = context.urlGraph.getUrlInfo(reference.parentUrl)
+    if (
+      !parentUrlInfo ||
+      !new URL(parentUrlInfo.url).searchParams.has("as_js_classic")
+    ) {
+      return null
+    }
+    const urlTransformed = injectQueryParams(reference.url, {
+      as_js_classic: "",
+    })
+    reference.filename = generateJsClassicFilename(reference.url)
+    return urlTransformed
+  }
+
+  return {
+    name: "jsenv:as_js_classic_conversion",
+    appliesDuring: "*",
+    redirectUrl: {
+      // We want to propagate transformation of js module to js classic to:
+      // - import specifier (static/dynamic import + re-export)
+      // - url specifier when inside System.register/_context.import()
+      //   (because it's the transpiled equivalent of static and dynamic imports)
+      // And not other references otherwise we could try to transform inline ressources
+      // or specifiers inside new URL()...
+      js_import_export: propagateJsClassicSearchParam,
+      js_url_specifier: (reference, context) => {
+        if (
+          reference.subtype === "system_register_arg" ||
+          reference.subtype === "system_import_arg"
+        ) {
+          return propagateJsClassicSearchParam(reference, context)
+        }
+        return null
+      },
+    },
+    fetchUrlContent: async (urlInfo, context) => {
+      const originalUrlInfo = await context.fetchOriginalUrlInfo({
+        urlInfo,
+        context,
+        searchParam: "as_js_classic",
+        // override the expectedType to "js_module"
+        // because when there is ?as_js_classic it means the underlying ressource
+        // is a js_module
+        expectedType: "js_module",
+      })
+      if (!originalUrlInfo) {
+        return null
+      }
+      const isJsEntryPoint =
+        // in general html files are entry points
+        // but during build js can be sepcified as an entry point
+        // (meaning there is no html file where we can inject systemjs)
+        // in that case we need to inject systemjs in the js file
+        originalUrlInfo.data.isEntryPoint ||
+        // In thoose case we need to inject systemjs the worker js file
+        originalUrlInfo.data.isWebWorkerEntryPoint
+      // if it's an entry point without dependency (it does not use import)
+      // then we can use UMD, otherwise we have to use systemjs
+      // because it is imported by systemjs
+      const jsClassicFormat =
+        isJsEntryPoint && !originalUrlInfo.data.usesImport ? "umd" : "system"
+      const { content, sourcemap } = await convertJsModuleToJsClassic({
+        systemJsInjection,
+        systemJsClientFileUrl,
+        urlInfo: originalUrlInfo,
+        isJsEntryPoint,
+        jsClassicFormat,
+      })
+      urlInfo.data.jsClassicFormat = jsClassicFormat
+      return {
+        originalUrl: originalUrlInfo.originalUrl,
+        originalContent: originalUrlInfo.originalContent,
+        type: "js_classic",
+        contentType: "text/javascript",
+        content,
+        sourcemap,
+      }
+    },
+  }
+}
+
+const generateJsClassicFilename = (url) => {
+  const filename = urlToFilename(url)
+  let [basename, extension] = splitFileExtension(filename)
+  const { searchParams } = new URL(url)
+  if (
+    searchParams.has("as_json_module") ||
+    searchParams.has("as_css_module") ||
+    searchParams.has("as_text_module")
+  ) {
+    extension = ".js"
+  }
+  return `${basename}.es5${extension}`
+}
+
+const splitFileExtension = (filename) => {
+  const dotLastIndex = filename.lastIndexOf(".")
+  if (dotLastIndex === -1) {
+    return [filename, ""]
+  }
+  return [filename.slice(0, dotLastIndex), filename.slice(dotLastIndex)]
+}
+
+const convertJsModuleToJsClassic = async ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+  urlInfo,
+  isJsEntryPoint,
+  jsClassicFormat,
+}) => {
+  const { code, map } = await applyBabelPlugins({
+    babelPlugins: [
+      ...(jsClassicFormat === "system"
+        ? [
+            // propposal-dynamic-import required with systemjs for babel8:
+            // https://github.com/babel/babel/issues/10746
+            require("@babel/plugin-proposal-dynamic-import"),
+            [
+              requireBabelPlugin("babel-plugin-transform-async-to-promises"),
+              {
+                topLevelAwait: "return",
+              },
+            ],
+            require("@babel/plugin-transform-modules-systemjs"),
+          ]
+        : [
+            [
+              requireBabelPlugin("babel-plugin-transform-async-to-promises"),
+              {
+                topLevelAwait: "simple",
+              },
+            ],
+            babelPluginTransformImportMetaUrl,
+            require("@babel/plugin-transform-modules-umd"),
+          ]),
+    ],
+    urlInfo,
+  })
+  let sourcemap = urlInfo.sourcemap
+  sourcemap = await composeTwoSourcemaps(sourcemap, map)
+  if (systemJsInjection && jsClassicFormat === "system" && isJsEntryPoint) {
+    const magicSource = createMagicSource(code)
+    const systemjsCode = readFileSync(systemJsClientFileUrl, { as: "string" })
+    magicSource.prepend(`${systemjsCode}\n\n`)
+    const magicResult = magicSource.toContentAndSourcemap()
+    sourcemap = await composeTwoSourcemaps(sourcemap, magicResult.sourcemap)
+    return {
+      content: magicResult.content,
+      sourcemap,
+    }
+  }
+  return {
+    content: code,
+    sourcemap,
+  }
+}

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic_html.js

@@ -0,0 +1,304 @@
+import {
+  getHtmlNodeAttributeByName,
+  getHtmlNodeTextNode,
+  parseHtmlString,
+  removeHtmlNodeAttributeByName,
+  assignHtmlNodeAttributes,
+  stringifyHtmlAst,
+  visitHtmlAst,
+  htmlNodePosition,
+  setHtmlNodeGeneratedText,
+  injectScriptAsEarlyAsPossible,
+  createHtmlNode,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import {
+  generateInlineContentUrl,
+  injectQueryParamsIntoSpecifier,
+} from "@jsenv/urls"
+
+export const jsenvPluginAsJsClassicHtml = ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+  generateJsClassicFilename,
+}) => {
+  return {
+    name: "jsenv:as_js_classic_html",
+    appliesDuring: "*",
+    transformUrlContent: {
+      html: async (urlInfo, context) => {
+        const shouldTransformScriptTypeModule =
+          !context.isSupportedOnCurrentClients("script_type_module") ||
+          !context.isSupportedOnCurrentClients("import_dynamic")
+        const htmlAst = parseHtmlString(urlInfo.content)
+        const preloadAsScriptNodes = []
+        const modulePreloadNodes = []
+        const moduleScriptNodes = []
+        const classicScriptNodes = []
+        const visitLinkNodes = (node) => {
+          if (node.nodeName !== "link") {
+            return
+          }
+          const relAttribute = getHtmlNodeAttributeByName(node, "rel")
+          const rel = relAttribute ? relAttribute.value : undefined
+          if (rel === "modulepreload") {
+            modulePreloadNodes.push(node)
+            return
+          }
+          if (rel === "preload") {
+            const asAttribute = getHtmlNodeAttributeByName(node, "as")
+            const asValue = asAttribute ? asAttribute.value : undefined
+            if (asValue === "script") {
+              preloadAsScriptNodes.push(node)
+            }
+            return
+          }
+        }
+        const visitScriptNodes = (node) => {
+          if (node.nodeName !== "script") {
+            return
+          }
+          const typeAttribute = getHtmlNodeAttributeByName(node, "type")
+          const type = typeAttribute ? typeAttribute.value : undefined
+          if (type === "module") {
+            moduleScriptNodes.push(node)
+            return
+          }
+          if (type === undefined || type === "text/javascript") {
+            classicScriptNodes.push(node)
+            return
+          }
+        }
+        visitHtmlAst(htmlAst, (node) => {
+          visitLinkNodes(node)
+          visitScriptNodes(node)
+        })
+
+        const actions = []
+        const jsModuleUrls = []
+        const convertedUrls = []
+        const getReferenceAsJsClassic = async (
+          reference,
+          {
+            // we don't cook ressource hints
+            // because they might refer to ressource that will be modified during build
+            // It also means something else HAVE to reference that url in order to cook it
+            // so that the preload is deleted by "resync_ressource_hints.js" otherwise
+            cookIt = false,
+          } = {},
+        ) => {
+          const newReferenceProps = {
+            expectedType: "js_classic",
+            specifier: injectQueryParamsIntoSpecifier(reference.specifier, {
+              as_js_classic: "",
+            }),
+            filename: generateJsClassicFilename(reference.url),
+          }
+          const [newReference, newUrlInfo] = context.referenceUtils.update(
+            reference,
+            newReferenceProps,
+          )
+          const convertedUrl = newUrlInfo.url
+          if (!convertedUrls.includes(convertedUrl)) {
+            convertedUrls.push(convertedUrl)
+          }
+          if (cookIt) {
+            // during dev it means js modules will be cooked before server sends the HTML
+            // it's ok because:
+            // - during dev script_type_module are supported (dev use a recent browser)
+            // - even if browser is not supported it still works it's jus a bit slower
+            //   because it needs to decide if systemjs will be injected or not
+            await context.cook(newUrlInfo, { reference: newReference })
+          }
+          return [newReference, newUrlInfo]
+        }
+
+        classicScriptNodes.forEach((classicScriptNode) => {
+          const srcAttribute = getHtmlNodeAttributeByName(
+            classicScriptNode,
+            "src",
+          )
+          if (srcAttribute) {
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === srcAttribute.value &&
+                ref.type === "script_src",
+            )
+            const urlObject = new URL(reference.url)
+            if (urlObject.searchParams.has("as_js_classic")) {
+              const convertedUrl = urlObject.href
+              convertedUrls.push(convertedUrl)
+              urlObject.searchParams.delete("as_js_classic")
+              const jsModuleUrl = urlObject.href
+              jsModuleUrls.push(jsModuleUrl)
+              actions.push(async () => {
+                const urlInfo = context.urlGraph.getUrlInfo(convertedUrl)
+                await context.cook(urlInfo, { reference })
+              })
+            }
+          }
+        })
+        moduleScriptNodes.forEach((moduleScriptNode) => {
+          const srcAttribute = getHtmlNodeAttributeByName(
+            moduleScriptNode,
+            "src",
+          )
+          if (srcAttribute) {
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === srcAttribute.value &&
+                ref.type === "script_src" &&
+                ref.expectedType === "js_module",
+            )
+            jsModuleUrls.push(reference.url)
+            if (shouldTransformScriptTypeModule) {
+              actions.push(async () => {
+                const [newReference] = await getReferenceAsJsClassic(
+                  reference,
+                  {
+                    cookIt: true,
+                  },
+                )
+                removeHtmlNodeAttributeByName(moduleScriptNode, "type")
+                srcAttribute.value = newReference.generatedSpecifier
+              })
+            }
+            return
+          }
+          if (shouldTransformScriptTypeModule) {
+            const textNode = getHtmlNodeTextNode(moduleScriptNode)
+            actions.push(async () => {
+              const { line, column, lineEnd, columnEnd, isOriginal } =
+                htmlNodePosition.readNodePosition(moduleScriptNode, {
+                  preferOriginal: true,
+                })
+              let inlineScriptUrl = generateInlineContentUrl({
+                url: urlInfo.url,
+                extension: ".js",
+                line,
+                column,
+                lineEnd,
+                columnEnd,
+              })
+              const [inlineReference] = context.referenceUtils.foundInline({
+                node: moduleScriptNode,
+                type: "script_src",
+                expectedType: "js_module",
+                isOriginalPosition: isOriginal,
+                // we remove 1 to the line because imagine the following html:
+                // <script>console.log('ok')</script>
+                // -> content starts same line as <script>
+                specifierLine: line - 1,
+                specifierColumn: column,
+                specifier: inlineScriptUrl,
+                contentType: "application/javascript",
+                content: textNode.value,
+              })
+              const [, newUrlInfo] = await getReferenceAsJsClassic(
+                inlineReference,
+                { cookIt: true },
+              )
+              removeHtmlNodeAttributeByName(moduleScriptNode, "type")
+              setHtmlNodeGeneratedText(moduleScriptNode, {
+                generatedText: newUrlInfo.content,
+                generatedBy: "jsenv:as_js_classic_html",
+              })
+            })
+          }
+        })
+        if (shouldTransformScriptTypeModule) {
+          preloadAsScriptNodes.forEach((preloadAsScriptNode) => {
+            const hrefAttribute = getHtmlNodeAttributeByName(
+              preloadAsScriptNode,
+              "href",
+            )
+            const href = hrefAttribute.value
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === href &&
+                ref.type === "link_href" &&
+                ref.expectedType === undefined,
+            )
+            const expectedScriptType = jsModuleUrls.includes(reference.url)
+              ? "module"
+              : "classic"
+            if (expectedScriptType === "module") {
+              actions.push(async () => {
+                // reference modified by <script type="module"> conversion
+                let newReference
+                if (reference.next) {
+                  newReference = reference.next
+                } else {
+                  // when the url is not referenced by a <script type="module">
+                  // we assume we want to preload "classic" but it might not be the case
+                  // but it's unlikely to happen and people should use "modulepreload" in that case anyway
+                  ;[newReference] = await getReferenceAsJsClassic(reference)
+                }
+                assignHtmlNodeAttributes(preloadAsScriptNode, {
+                  href: newReference.generatedSpecifier,
+                })
+                removeHtmlNodeAttributeByName(
+                  preloadAsScriptNode,
+                  "crossorigin",
+                )
+              })
+            }
+          })
+          modulePreloadNodes.forEach((modulePreloadNode) => {
+            const hrefAttribute = getHtmlNodeAttributeByName(
+              modulePreloadNode,
+              "href",
+            )
+            const href = hrefAttribute.value
+            const reference = urlInfo.references.find(
+              (ref) =>
+                ref.generatedSpecifier === href &&
+                ref.type === "link_href" &&
+                ref.expectedType === "js_module",
+            )
+            actions.push(async () => {
+              let newReference
+              if (reference.next) {
+                newReference = reference.next
+              } else {
+                ;[newReference] = await getReferenceAsJsClassic(reference)
+              }
+              assignHtmlNodeAttributes(modulePreloadNode, {
+                rel: "preload",
+                as: "script",
+                href: newReference.generatedSpecifier,
+              })
+            })
+          })
+        }
+
+        if (actions.length === 0) {
+          return null
+        }
+        await Promise.all(actions.map((action) => action()))
+        if (systemJsInjection) {
+          const needsSystemJs = convertedUrls.some(
+            (convertedUrl) =>
+              context.urlGraph.getUrlInfo(convertedUrl).data.jsClassicFormat ===
+              "system",
+          )
+          if (needsSystemJs) {
+            const [systemJsReference] = context.referenceUtils.inject({
+              type: "script_src",
+              expectedType: "js_classic",
+              specifier: systemJsClientFileUrl,
+            })
+            injectScriptAsEarlyAsPossible(
+              htmlAst,
+              createHtmlNode({
+                "tagName": "script",
+                "src": systemJsReference.generatedSpecifier,
+                "injected-by": "jsenv:as_js_classic_html",
+              }),
+            )
+          }
+        }
+        return stringifyHtmlAst(htmlAst)
+      },
+    },
+  }
+}

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic_workers.js

@@ -0,0 +1,55 @@
+import { injectQueryParams } from "@jsenv/urls"
+
+export const jsenvPluginAsJsClassicWorkers = ({
+  generateJsClassicFilename,
+}) => {
+  const updateReference = (reference) => {
+    reference.filename = generateJsClassicFilename(reference.url)
+    reference.mutation = (magicSource) => {
+      magicSource.replace({
+        start: reference.typePropertyNode.value.start,
+        end: reference.typePropertyNode.value.end,
+        replacement: JSON.stringify("classic"),
+      })
+    }
+    reference.expectedType = "js_classic"
+    return injectQueryParams(reference.url, {
+      as_js_classic: "",
+    })
+  }
+
+  return {
+    name: "jsenv:as_js_classic_workers",
+    appliesDuring: "*",
+    redirectUrl: {
+      js_url_specifier: (reference, context) => {
+        if (reference.expectedType !== "js_module") {
+          return null
+        }
+        if (reference.expectedSubtype === "worker") {
+          if (context.isSupportedOnCurrentClients("worker_type_module")) {
+            return null
+          }
+          return updateReference(reference)
+        }
+        if (reference.expectedSubtype === "service_worker") {
+          if (
+            context.isSupportedOnCurrentClients("service_worker_type_module")
+          ) {
+            return null
+          }
+          return updateReference(reference)
+        }
+        if (reference.expectedSubtype === "shared_worker") {
+          if (
+            context.isSupportedOnCurrentClients("shared_worker_type_module")
+          ) {
+            return null
+          }
+          return updateReference(reference)
+        }
+        return null
+      },
+    },
+  }
+}

package/src/{omega/core_plugins → plugins/transpilation}/babel/helpers/babel_plugin_structure.js

@@ -3,14 +3,12 @@ import { babelPluginCompatMap } from "./babel_plugins_compatibility.js"
 
 export const getBaseBabelPluginStructure = ({
   url,
-  isSupportedOnRuntime,
-  usesTopLevelAwait,
-  //   // https://github.com/rpetrich/babel-plugin-transform-async-to-promises/blob/92755ff8c943c97596523e586b5fa515c2e99326/async-to-promises.ts#L55
-  topLevelAwait = "simple",
-  isJsModule,
+  isSupported,
+  // isJsModule,
+  // getImportSpecifier,
 }) => {
   const isBabelPluginNeeded = (babelPluginName) => {
-    return !isSupportedOnRuntime(babelPluginCompatMap[babelPluginName])
+    return !isSupported(babelPluginCompatMap[babelPluginName])
   }
 
   const babelPluginStructure = {}
@@ -37,21 +35,19 @@ export const getBaseBabelPluginStructure = ({
     babelPluginStructure["proposal-unicode-property-regex"] =
       requireBabelPlugin("@babel/plugin-proposal-unicode-property-regex")
   }
-  if (
-    isJsModule &&
-    usesTopLevelAwait &&
-    !isSupportedOnRuntime("top_level_await")
-  ) {
+  if (isBabelPluginNeeded("transform-async-to-promises")) {
     babelPluginStructure["transform-async-to-promises"] = [
       requireBabelPlugin("babel-plugin-transform-async-to-promises"),
       {
-        topLevelAwait,
+        topLevelAwait: "ignore", // will be handled by "jsenv:top_level_await" plugin
+        externalHelpers: false,
+        // enable once https://github.com/rpetrich/babel-plugin-transform-async-to-promises/pull/83
+        // externalHelpers: isJsModule,
+        // externalHelpersPath: isJsModule ? getImportSpecifier(
+        //     "babel-plugin-transform-async-to-promises/helpers.mjs",
+        //   ) : null
       },
     ]
-  } else if (isBabelPluginNeeded("transform-async-to-promises")) {
-    babelPluginStructure["transform-async-to-promises"] = requireBabelPlugin(
-      "babel-plugin-transform-async-to-promises",
-    )
   }
   if (isBabelPluginNeeded("transform-arrow-functions")) {
     babelPluginStructure["transform-arrow-functions"] = requireBabelPlugin(