@jsenv/core 27.0.0-alpha.12 → 27.0.0-alpha.15

package/src/plugins/transpilation/as_js_classic/client/s.js.md

@@ -0,0 +1 @@
+s.js exists to ease jsenv ability to find the file.

package/src/plugins/transpilation/as_js_classic/helpers/babel_plugin_transform_import_meta_url.js

@@ -0,0 +1,47 @@
+import babelParser from "@babel/parser"
+
+export const babelPluginTransformImportMetaUrl = (babel) => {
+  return {
+    name: "transform-import-meta-url",
+    visitor: {
+      Program: (programPath) => {
+        const currentUrlIdentifier =
+          programPath.scope.generateUidIdentifier("currentUrl")
+        let used = false
+
+        programPath.traverse({
+          MemberExpression: (path) => {
+            const node = path.node
+            if (
+              node.object.type === "MetaProperty" &&
+              node.object.property.name === "meta" &&
+              node.property.name === "url"
+            ) {
+              // const node = babel.types.valueToNode(10)
+              const identifier = babel.types.identifier(
+                currentUrlIdentifier.name,
+              )
+              const expressionStatement =
+                babel.types.expressionStatement(identifier)
+              path.replaceWith(expressionStatement)
+              used = true
+            }
+          },
+        })
+        if (used) {
+          const ast = generateExpressionAst(`document.currentScript.src`)
+          programPath.scope.push({
+            id: currentUrlIdentifier,
+            init: ast,
+          })
+        }
+      },
+    },
+  }
+}
+
+const generateExpressionAst = (expression, options) => {
+  const { parseExpression } = babelParser
+  const ast = parseExpression(expression, options)
+  return ast
+}

package/src/plugins/transpilation/as_js_classic/helpers/systemjs_old.js

@@ -0,0 +1,43 @@
+// const { clientRuntimeCompat, isSupportedOnCurrentClient } = context
+// const shouldBeCompatibleWithNode =
+//   Object.keys(clientRuntimeCompat).includes("node")
+// const requiredFeatureNames = [
+//   "import_dynamic",
+//   "top_level_await",
+//   "global_this",
+//   // when using node we assume the code won't use browser specific feature
+//   ...(shouldBeCompatibleWithNode
+//     ? []
+//     : [
+//         "script_type_module",
+//         "worker_type_module",
+//         "import_type_json",
+//         "import_type_css",
+//       ]),
+//   // "importmap",
+// ]
+// const needsSystemJs = featuresRelatedToSystemJs.some((featureName) => {
+//   const isRequired = requiredFeatureNames.includes(featureName)
+//   return isRequired && !isSupportedOnCurrentClient(featureName)
+// })
+// if (!needsSystemJs) {
+//   return null
+// }
+// const { code, map } = await applyBabelPlugins({
+
+// })
+// return {
+//   content: code,
+//   soourcemap: map,
+// }
+
+// const featuresRelatedToSystemJs = [
+//   "script_type_module",
+//   "worker_type_module",
+//   "import_dynamic",
+//   "import_type_json",
+//   "import_type_css",
+//   "top_level_await",
+//   // "importmap",
+//   // "worker_importmap",
+// ]

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_as_js_classic.js

@@ -0,0 +1,191 @@
+/*
+ * Something to keep in mind:
+ * When systemjs format is used by babel, it will generated UID based on
+ * the import specifier:
+ * https://github.com/babel/babel/blob/97d1967826077f15e766778c0d64711399e9a72a/packages/babel-plugin-transform-modules-systemjs/src/index.ts#L498
+ * But at this stage import specifier are absolute file urls
+ * So without minification these specifier are long and dependent
+ * on where the files are on the filesystem.
+ * This is mitigated by minification that will shorten them
+ * But ideally babel should not generate this in the first place
+ * and prefer to unique identifier based solely on the specifier basename for instance
+ */
+
+import { createRequire } from "node:module"
+import { readFileSync, urlToFilename } from "@jsenv/filesystem"
+
+import { requireBabelPlugin } from "@jsenv/babel-plugins"
+
+import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
+import { injectQueryParams } from "@jsenv/utils/urls/url_utils.js"
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+import { babelPluginTransformImportMetaUrl } from "./helpers/babel_plugin_transform_import_meta_url.js"
+import { jsenvPluginScriptTypeModuleAsClassic } from "./jsenv_plugin_script_type_module_as_classic.js"
+import { jsenvPluginWorkersTypeModuleAsClassic } from "./jsenv_plugin_workers_type_module_as_classic.js"
+import { jsenvPluginTopLevelAwait } from "./jsenv_plugin_top_level_await.js"
+
+const require = createRequire(import.meta.url)
+
+export const jsenvPluginAsJsClassic = ({ systemJsInjection }) => {
+  const systemJsClientFileUrl = new URL("./client/s.js", import.meta.url).href
+
+  return [
+    asJsClassic({ systemJsInjection, systemJsClientFileUrl }),
+    jsenvPluginScriptTypeModuleAsClassic({
+      systemJsInjection,
+      systemJsClientFileUrl,
+      generateJsClassicFilename,
+    }),
+    jsenvPluginWorkersTypeModuleAsClassic({
+      generateJsClassicFilename,
+    }),
+    jsenvPluginTopLevelAwait(),
+  ]
+}
+
+const asJsClassic = ({ systemJsInjection, systemJsClientFileUrl }) => {
+  return {
+    name: "jsenv:as_js_classic",
+    appliesDuring: "*",
+    // forward ?as_js_classic to referenced urls
+    normalizeUrl: (reference, context) => {
+      if (reference.isInline) {
+        if (reference.contentType !== "text/javascript") {
+          // We want to propagate transformation of js module to js classic
+          // so we don't want to propagate when the reference is not js:
+          // - ignore "string" in `JSON.parse(string)` for instance
+          return null
+        }
+      }
+      const parentUrlInfo = context.urlGraph.getUrlInfo(reference.parentUrl)
+      if (
+        !parentUrlInfo ||
+        !new URL(parentUrlInfo.url).searchParams.has("as_js_classic")
+      ) {
+        return null
+      }
+      const urlTransformed = injectQueryParams(reference.url, {
+        as_js_classic: "",
+      })
+      reference.filename = generateJsClassicFilename(reference.url)
+      return urlTransformed
+    },
+    fetchUrlContent: async (urlInfo, context) => {
+      const urlObject = new URL(urlInfo.url)
+      const { searchParams } = urlObject
+      if (!searchParams.has("as_js_classic")) {
+        return null
+      }
+      searchParams.delete("as_js_classic")
+      const originalUrl = urlObject.href
+      const originalReference = {
+        ...(context.reference.original || context.reference),
+        // override the expectedType to "js_module"
+        // because when there is ?as_js_classic it means the underlying ressource
+        // is a js_module
+        expectedType: "js_module",
+      }
+      originalReference.url = originalUrl
+      const originalUrlInfo = context.urlGraph.reuseOrCreateUrlInfo(
+        originalReference.url,
+      )
+      await context.fetchUrlContent({
+        reference: originalReference,
+        urlInfo: originalUrlInfo,
+      })
+      const isJsEntryPoint =
+        // in general html files are entry points
+        // but during build js can be sepcified as an entry point
+        // (meaning there is no html file where we can inject systemjs)
+        // in that case we need to inject systemjs in the js file
+        originalUrlInfo.data.isEntryPoint ||
+        // In thoose case we need to inject systemjs the worker js file
+        originalUrlInfo.data.isWebWorkerEntryPoint
+      // if it's an entry point without dependency (it does not use import)
+      // then we can use UMD, otherwise we have to use systemjs
+      // because it is imported by systemjs
+      const jsClassicFormat =
+        isJsEntryPoint && !originalUrlInfo.data.usesImport ? "umd" : "system"
+      const { content, sourcemap } = await convertJsModuleToJsClassic({
+        systemJsInjection,
+        systemJsClientFileUrl,
+        urlInfo: originalUrlInfo,
+        isJsEntryPoint,
+        jsClassicFormat,
+      })
+      urlInfo.data.jsClassicFormat = jsClassicFormat
+      return {
+        type: "js_classic",
+        contentType: "text/javascript",
+        content,
+        sourcemap,
+      }
+    },
+  }
+}
+
+const generateJsClassicFilename = (url) => {
+  const filename = urlToFilename(url)
+  const [basename, extension] = splitFileExtension(filename)
+  return `${basename}.es5${extension}`
+}
+
+const splitFileExtension = (filename) => {
+  const dotLastIndex = filename.lastIndexOf(".")
+  if (dotLastIndex === -1) {
+    return [filename, ""]
+  }
+  return [filename.slice(0, dotLastIndex), filename.slice(dotLastIndex)]
+}
+
+const convertJsModuleToJsClassic = async ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+  urlInfo,
+  isJsEntryPoint,
+  jsClassicFormat,
+}) => {
+  const { code, map } = await applyBabelPlugins({
+    babelPlugins: [
+      ...(jsClassicFormat === "system"
+        ? [
+            // propposal-dynamic-import required with systemjs for babel8:
+            // https://github.com/babel/babel/issues/10746
+            require("@babel/plugin-proposal-dynamic-import"),
+            [
+              requireBabelPlugin("babel-plugin-transform-async-to-promises"),
+              {
+                topLevelAwait: "return",
+              },
+            ],
+            require("@babel/plugin-transform-modules-systemjs"),
+          ]
+        : [
+            [
+              requireBabelPlugin("babel-plugin-transform-async-to-promises"),
+              {
+                topLevelAwait: "simple",
+              },
+            ],
+            babelPluginTransformImportMetaUrl,
+            require("@babel/plugin-transform-modules-umd"),
+          ]),
+    ],
+    urlInfo,
+  })
+  if (systemJsInjection && jsClassicFormat === "system" && isJsEntryPoint) {
+    const magicSource = createMagicSource(code)
+    const systemjsCode = readFileSync(systemJsClientFileUrl, { as: "string" })
+    magicSource.prepend(`${systemjsCode}\n\n`)
+    const { content, sourcemap } = magicSource.toContentAndSourcemap()
+    return {
+      content,
+      sourcemap: await composeTwoSourcemaps(map, sourcemap),
+    }
+  }
+  return {
+    content: code,
+    sourcemap: map,
+  }
+}

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_script_type_module_as_classic.js

@@ -0,0 +1,156 @@
+import {
+  getHtmlNodeAttributeByName,
+  getHtmlNodeTextNode,
+  parseHtmlString,
+  removeHtmlNodeAttribute,
+  stringifyHtmlAst,
+  visitHtmlAst,
+  htmlNodePosition,
+  setHtmlNodeGeneratedText,
+  injectScriptAsEarlyAsPossible,
+  createHtmlNode,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import { generateInlineContentUrl } from "@jsenv/utils/urls/inline_content_url_generator.js"
+import { injectQueryParamsIntoSpecifier } from "@jsenv/utils/urls/url_utils.js"
+
+export const jsenvPluginScriptTypeModuleAsClassic = ({
+  systemJsInjection,
+  systemJsClientFileUrl,
+  generateJsClassicFilename,
+}) => {
+  return {
+    name: "jsenv:script_type_module_as_classic",
+    appliesDuring: "*",
+    transformUrlContent: {
+      html: async (urlInfo, context) => {
+        if (
+          context.isSupportedOnCurrentClients("script_type_module") &&
+          context.isSupportedOnCurrentClients("import_dynamic")
+        ) {
+          return null
+        }
+        const htmlAst = parseHtmlString(urlInfo.content)
+        const actions = []
+        const jsModuleUrlInfos = []
+        const visitScriptTypeModule = (node) => {
+          if (node.nodeName !== "script") {
+            return
+          }
+          const typeAttribute = getHtmlNodeAttributeByName(node, "type")
+          if (!typeAttribute || typeAttribute.value !== "module") {
+            return
+          }
+          const srcAttribute = getHtmlNodeAttributeByName(node, "src")
+          if (srcAttribute) {
+            actions.push(async () => {
+              const specifier = srcAttribute.value
+              const reference =
+                context.referenceUtils.findByGeneratedSpecifier(specifier)
+              const [newReference, newUrlInfo] = context.referenceUtils.update(
+                reference,
+                {
+                  expectedType: "js_classic",
+                  specifier: injectQueryParamsIntoSpecifier(specifier, {
+                    as_js_classic: "",
+                  }),
+                  filename: generateJsClassicFilename(reference.url),
+                },
+              )
+              removeHtmlNodeAttribute(node, typeAttribute)
+              srcAttribute.value = newReference.generatedSpecifier
+              // during dev it means js modules will be cooked before server sends the HTML
+              // it's ok because:
+              // - during dev script_type_module are supported (dev use a recent browser)
+              // - even if browser is not supported it still works it's jus a bit slower
+              //   because it needs to decide if systemjs will be injected or not
+              await context.cook({
+                reference: newReference,
+                urlInfo: newUrlInfo,
+              })
+              jsModuleUrlInfos.push(newUrlInfo)
+            })
+            return
+          }
+          const textNode = getHtmlNodeTextNode(node)
+          actions.push(async () => {
+            const { line, column, lineEnd, columnEnd, isOriginal } =
+              htmlNodePosition.readNodePosition(node, {
+                preferOriginal: true,
+              })
+            let inlineScriptUrl = generateInlineContentUrl({
+              url: urlInfo.url,
+              extension: ".js",
+              line,
+              column,
+              lineEnd,
+              columnEnd,
+            })
+            const [inlineReference] = context.referenceUtils.foundInline({
+              node,
+              type: "script_src",
+              expectedType: "js_module",
+              // we remove 1 to the line because imagine the following html:
+              // <script>console.log('ok')</script>
+              // -> content starts same line as <script>
+              line: line - 1,
+              column,
+              isOriginalPosition: isOriginal,
+              specifier: inlineScriptUrl,
+              contentType: "application/javascript",
+              content: textNode.value,
+            })
+            const [newReference, newUrlInfo] = context.referenceUtils.update(
+              inlineReference,
+              {
+                expectedType: "js_classic",
+                specifier: injectQueryParamsIntoSpecifier(inlineScriptUrl, {
+                  as_js_classic: "",
+                }),
+                filename: generateJsClassicFilename(inlineReference.url),
+              },
+            )
+            await context.cook({
+              reference: newReference,
+              urlInfo: newUrlInfo,
+            })
+            removeHtmlNodeAttribute(node, typeAttribute)
+            setHtmlNodeGeneratedText(node, {
+              generatedText: newUrlInfo.content,
+              generatedBy: "jsenv:script_type_module_as_classic",
+            })
+            jsModuleUrlInfos.push(newUrlInfo)
+          })
+        }
+        visitHtmlAst(htmlAst, (node) => {
+          visitScriptTypeModule(node)
+        })
+        if (actions.length === 0) {
+          return null
+        }
+        await Promise.all(actions.map((action) => action()))
+        if (systemJsInjection) {
+          const needsSystemJs = jsModuleUrlInfos.some(
+            (jsModuleUrlInfo) =>
+              jsModuleUrlInfo.data.jsClassicFormat === "system",
+          )
+          if (needsSystemJs) {
+            const [systemJsReference] = context.referenceUtils.inject({
+              type: "script_src",
+              expectedType: "js_classic",
+              specifier: systemJsClientFileUrl,
+            })
+            injectScriptAsEarlyAsPossible(
+              htmlAst,
+              createHtmlNode({
+                "tagName": "script",
+                "src": systemJsReference.generatedSpecifier,
+                "injected-by": "jsenv:script_type_module_as_classic",
+              }),
+            )
+          }
+        }
+        return stringifyHtmlAst(htmlAst)
+      },
+    },
+  }
+}

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_top_level_await.js

@@ -0,0 +1,37 @@
+import { requireBabelPlugin } from "@jsenv/babel-plugins"
+
+import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
+
+export const jsenvPluginTopLevelAwait = () => {
+  return {
+    name: "jsenv:top_level_await",
+    appliesDuring: "*",
+    transformUrlContent: {
+      js_module: async (urlInfo, context) => {
+        if (!urlInfo.data.usesTopLevelAwait) {
+          return null
+        }
+        if (context.isSupportedOnCurrentClients("top_level_await")) {
+          return null
+        }
+        const { code, map } = await applyBabelPlugins({
+          urlInfo,
+          babelPlugins: [
+            [
+              requireBabelPlugin("babel-plugin-transform-async-to-promises"),
+              {
+                // Maybe we could pass target: "es6" when we support arrow function
+                // https://github.com/rpetrich/babel-plugin-transform-async-to-promises/blob/92755ff8c943c97596523e586b5fa515c2e99326/async-to-promises.ts#L55
+                topLevelAwait: "simple",
+              },
+            ],
+          ],
+        })
+        return {
+          content: code,
+          sourcemap: map,
+        }
+      },
+    },
+  }
+}

package/src/plugins/transpilation/as_js_classic/jsenv_plugin_workers_type_module_as_classic.js

@@ -0,0 +1,133 @@
+import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
+import { injectQueryParamsIntoSpecifier } from "@jsenv/utils/urls/url_utils.js"
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+import {
+  analyzeNewWorkerOrNewSharedWorker,
+  analyzeServiceWorkerRegisterCall,
+} from "@jsenv/utils/js_ast/js_static_analysis.js"
+
+// TODO: handle also service worker and shared worker in this plugin
+export const jsenvPluginWorkersTypeModuleAsClassic = ({
+  generateJsClassicFilename,
+}) => {
+  const transformJsWorkerTypes = async (urlInfo, context) => {
+    const workersToTranspile = getWorkersToTranspile(urlInfo, context)
+    if (
+      !workersToTranspile.worker &&
+      !workersToTranspile.serviceWorker &&
+      !workersToTranspile.sharedServiceWorker
+    ) {
+      return null
+    }
+    const { metadata } = await applyBabelPlugins({
+      babelPlugins: [
+        [
+          babelPluginMetadataWorkerMentions,
+          {
+            workersToTranspile,
+          },
+        ],
+      ],
+      urlInfo,
+    })
+    const { workerMentions } = metadata
+    const magicSource = createMagicSource(urlInfo.content)
+    workerMentions.forEach((workerMention) => {
+      if (workerMention.expectedType !== "js_module") {
+        return
+      }
+      const specifier = workerMention.specifier
+      const reference = context.referenceUtils.findByGeneratedSpecifier(
+        JSON.stringify(specifier),
+      )
+      const [newReference] = context.referenceUtils.update(reference, {
+        expectedType: "js_classic",
+        specifier: injectQueryParamsIntoSpecifier(specifier, {
+          as_js_classic: "",
+        }),
+        filename: generateJsClassicFilename(reference.url),
+      })
+      magicSource.replace({
+        start: workerMention.start,
+        end: workerMention.end,
+        replacement: newReference.generatedSpecifier,
+      })
+      magicSource.replace({
+        start: workerMention.typeArgNode.value.start,
+        end: workerMention.typeArgNode.value.end,
+        replacement: JSON.stringify("classic"),
+      })
+    })
+    return magicSource.toContentAndSourcemap()
+  }
+
+  return {
+    name: "jsenv:workers_type_module_as_classic",
+    appliesDuring: "*",
+    transformUrlContent: {
+      js_module: transformJsWorkerTypes,
+      js_classic: transformJsWorkerTypes,
+    },
+  }
+}
+
+const getWorkersToTranspile = (urlInfo, context) => {
+  let worker = false
+  let serviceWorker = false
+  let sharedWorker = false
+  for (const reference of urlInfo.references) {
+    if (reference.expectedType !== "js_module") {
+      continue
+    }
+    if (
+      reference.expectedSubtype === "worker" &&
+      !context.isSupportedOnCurrentClients("worker_type_module")
+    ) {
+      worker = true
+    }
+    if (
+      reference.expectedSubtype === "service_worker" &&
+      !context.isSupportedOnCurrentClients("service_worker_type_module")
+    ) {
+      serviceWorker = true
+    }
+    if (
+      reference.expectedSubtype === "shared_worker" &&
+      !context.isSupportedOnCurrentClients("shared_worker_type_module")
+    ) {
+      sharedWorker = true
+    }
+  }
+  return { worker, serviceWorker, sharedWorker }
+}
+
+const babelPluginMetadataWorkerMentions = (_, { workersToTranspile }) => {
+  return {
+    name: "metadata-worker-mentions",
+    visitor: {
+      Program(programPath, state) {
+        const workerMentions = []
+        const visitors = {
+          NewExpression: (path) => {
+            if (workersToTranspile.worker || workersToTranspile.sharedWorker) {
+              const mentions = analyzeNewWorkerOrNewSharedWorker(path)
+              if (mentions) {
+                workerMentions.push(...mentions)
+              }
+            }
+          },
+        }
+        if (workersToTranspile.serviceWorker) {
+          visitors.CallExpression = (path) => {
+            const mentions = analyzeServiceWorkerRegisterCall(path)
+            if (mentions) {
+              workerMentions.push(...mentions)
+            }
+          }
+        }
+        programPath.traverse(visitors)
+        state.file.metadata.workerMentions = workerMentions
+      },
+    },
+  }
+}

package/src/{omega/core_plugins → plugins/transpilation}/babel/helpers/babel_plugin_structure.js

@@ -1,16 +1,9 @@
 import { getBabelHelperFileUrl, requireBabelPlugin } from "@jsenv/babel-plugins"
 import { babelPluginCompatMap } from "./babel_plugins_compatibility.js"
 
-export const getBaseBabelPluginStructure = ({
-  url,
-  isSupportedOnRuntime,
-  usesTopLevelAwait,
-  //   // https://github.com/rpetrich/babel-plugin-transform-async-to-promises/blob/92755ff8c943c97596523e586b5fa515c2e99326/async-to-promises.ts#L55
-  topLevelAwait = "simple",
-  isJsModule,
-}) => {
+export const getBaseBabelPluginStructure = ({ url, isSupported }) => {
   const isBabelPluginNeeded = (babelPluginName) => {
-    return !isSupportedOnRuntime(babelPluginCompatMap[babelPluginName])
+    return !isSupported(babelPluginCompatMap[babelPluginName])
   }
 
   const babelPluginStructure = {}
@@ -37,18 +30,7 @@ export const getBaseBabelPluginStructure = ({
     babelPluginStructure["proposal-unicode-property-regex"] =
       requireBabelPlugin("@babel/plugin-proposal-unicode-property-regex")
   }
-  if (
-    isJsModule &&
-    usesTopLevelAwait &&
-    !isSupportedOnRuntime("top_level_await")
-  ) {
-    babelPluginStructure["transform-async-to-promises"] = [
-      requireBabelPlugin("babel-plugin-transform-async-to-promises"),
-      {
-        topLevelAwait,
-      },
-    ]
-  } else if (isBabelPluginNeeded("transform-async-to-promises")) {
+  if (isBabelPluginNeeded("transform-async-to-promises")) {
     babelPluginStructure["transform-async-to-promises"] = requireBabelPlugin(
       "babel-plugin-transform-async-to-promises",
     )