@jsenv/core 24.6.2 → 25.0.0-alpha.0

package/src/internal/building/url_loader.js

@@ -1,17 +1,12 @@
-import { resolveUrl, urlToFilename } from "@jsenv/filesystem"
-
-import { transformJs } from "@jsenv/core/src/internal/compiling/js-compilation-service/transformJs.js"
-import { convertCssTextToJavascriptModule } from "@jsenv/core/src/internal/building/css_module.js"
+import { convertJsonTextToJavascriptModule } from "@jsenv/core/src/internal/building/json_module.js"
 import { getJavaScriptSourceMappingUrl } from "@jsenv/core/src/internal/sourceMappingURLUtils.js"
+
 import { loadSourcemap } from "./sourcemap_loader.js"
 
 export const createUrlLoader = ({
-  projectDirectoryUrl,
-  buildDirectoryUrl,
-  babelPluginMap,
+  urlCustomLoaders,
   allowJson,
   urlImporterMap,
-  inlineModuleScripts,
 
   asServerUrl,
   asProjectUrl,
@@ -21,112 +16,13 @@ export const createUrlLoader = ({
 }) => {
   const urlResponseBodyMap = {}
 
-  const loadUrl = async (rollupUrl, { signal, logger, ressourceBuilder }) => {
+  const loadUrl = async (rollupUrl, { signal, logger }) => {
     let url = asServerUrl(rollupUrl)
-    const { importType, urlWithoutImportType } = extractImportTypeFromUrl(url)
-    // importing CSS from JS with import assertions
-    if (importType === "css") {
-      const importer = urlImporterMap[url]
-      const cssReference =
-        await ressourceBuilder.createReferenceFoundInJsModule({
-          referenceLabel: "css import assertion",
-          // If all references to a ressource are only import assertions
-          // the file referenced do not need to be written on filesystem
-          // as it was converted to a js file
-          // We pass "isImportAssertion: true" for this purpose
-          isImportAssertion: true,
-          jsUrl: importer.url,
-          jsLine: importer.line,
-          jsColumn: importer.column,
-          ressourceSpecifier: urlWithoutImportType,
-          contentTypeExpected: "text/css",
-        })
-      await cssReference.ressource.getReadyPromise()
-      const cssBuildUrl = resolveUrl(
-        cssReference.ressource.buildRelativeUrl,
-        buildDirectoryUrl,
-      )
-      const jsBuildUrl = resolveUrl(
-        urlToFilename(importer.url),
-        buildDirectoryUrl,
-      )
-      let code = String(cssReference.ressource.bufferAfterBuild)
-      let map
-      const sourcemapReference = cssReference.ressource.dependencies.find(
-        (dependency) => {
-          return dependency.ressource.isSourcemap
-        },
-      )
-      if (sourcemapReference) {
-        // because css is ready, it's sourcemap is also ready
-        // we can read directly sourcemapReference.ressource.bufferAfterBuild
-        map = JSON.parse(sourcemapReference.ressource.bufferAfterBuild)
-      }
-
-      const jsModuleConversionResult = await convertCssTextToJavascriptModule({
-        cssUrl: cssBuildUrl,
-        jsUrl: jsBuildUrl,
-        code,
-        map,
-      })
-      code = jsModuleConversionResult.code
-      map = jsModuleConversionResult.map
-
-      return {
-        url,
-        code,
-        map,
-      }
-    }
-    // importing json from JS with import assertion
-    if (importType === "json") {
-      const importer = urlImporterMap[url]
-      const jsonReference =
-        await ressourceBuilder.createReferenceFoundInJsModule({
-          referenceLabel: "json import assertion",
-          // If all references to a ressource are only import assertions
-          // the file referenced do not need to be written on filesystem
-          // as it was converted to a js file
-          // We pass "isImportAssertion: true" for this purpose
-          isImportAssertion: true,
-          jsUrl: importer.url,
-          jsLine: importer.line,
-          jsColumn: importer.column,
-          ressourceSpecifier: asServerUrl(urlWithoutImportType),
-          contentTypeExpected: "application/json",
-        })
-      await jsonReference.ressource.getReadyPromise()
-      let code = String(jsonReference.ressource.bufferAfterBuild)
-      let map
-
-      const jsModuleConversionResult = await convertJsonTextToJavascriptModule({
-        code,
-        map,
-      })
-      code = jsModuleConversionResult.code
-      map = jsModuleConversionResult.map
 
-      return {
-        url,
-        code,
-      }
-    }
-
-    if (url in inlineModuleScripts) {
-      const transformResult = await transformJs({
-        code: String(inlineModuleScripts[url].bufferBeforeBuild),
-        url: asOriginalUrl(url), // transformJs expect a file:// url
-        projectDirectoryUrl,
-        babelPluginMap,
-        // moduleOutFormat: format // we are compiling for rollup output must be "esmodule"
-      })
-      let code = transformResult.code
-      let map = transformResult.map
-      return {
-        url,
-        code,
-        map,
-      }
+    const customLoader = urlCustomLoaders[url]
+    if (customLoader) {
+      const result = await customLoader()
+      return result
     }
 
     const response = await urlFetcher.fetchUrl(url, {
@@ -233,33 +129,3 @@ export const createUrlLoader = ({
     getUrlResponseBodyMap: () => urlResponseBodyMap,
   }
 }
-
-const extractImportTypeFromUrl = (url) => {
-  const urlObject = new URL(url)
-  const { search } = urlObject
-  const searchParams = new URLSearchParams(search)
-
-  const importType = searchParams.get("import_type")
-  if (!importType) {
-    return {}
-  }
-
-  searchParams.delete("import_type")
-  urlObject.search = String(searchParams)
-  return {
-    importType,
-    urlWithoutImportType: urlObject.href,
-  }
-}
-
-const convertJsonTextToJavascriptModule = ({ code }) => {
-  // here we could do the following
-  // return export default jsonText
-  // This would return valid js, that would be minified later
-  // however we will prefer using JSON.parse because it's faster
-  // for js engine to parse JSON than JS
-
-  return {
-    code: `export default JSON.parse(${JSON.stringify(code.trim())})`,
-  }
-}

package/src/internal/building/url_versioning.js

@@ -0,0 +1,226 @@
+import { createHash } from "crypto"
+import {
+  urlToParentUrl,
+  urlToBasename,
+  urlToExtension,
+} from "@jsenv/filesystem"
+
+export const createUrlVersioner = ({
+  entryPointUrls,
+  workerUrls,
+  classicWorkerUrls,
+  serviceWorkerUrls,
+  classicServiceWorkerUrls,
+  asOriginalUrl,
+  lineBreakNormalization,
+}) => {
+  const computeBuildRelativeUrl = (ressource) => {
+    const pattern = getFilenamePattern({
+      ressource,
+      entryPointUrls,
+      workerUrls,
+      classicWorkerUrls,
+      serviceWorkerUrls,
+      classicServiceWorkerUrls,
+      asOriginalUrl,
+      precomputeBuildRelativeUrl,
+    })
+    const buildRelativeUrl = generateBuildRelativeUrl(
+      ressource.url,
+      ressource.bufferAfterBuild,
+      {
+        pattern,
+        contentType: ressource.contentType,
+        lineBreakNormalization,
+      },
+    )
+    return buildRelativeUrl
+  }
+
+  const precomputeBuildRelativeUrl = (ressource, bufferAfterBuild = "") => {
+    if (ressource.buildRelativeUrl) {
+      return ressource.buildRelativeUrl
+    }
+    if (ressource.precomputedBuildRelativeUrl) {
+      return ressource.precomputedBuildRelativeUrl
+    }
+
+    ressource.bufferAfterBuild = bufferAfterBuild
+    const precomputedBuildRelativeUrl = computeBuildRelativeUrl(ressource)
+    ressource.bufferAfterBuild = undefined
+    ressource.precomputedBuildRelativeUrl = precomputedBuildRelativeUrl
+    return precomputedBuildRelativeUrl
+  }
+
+  return {
+    computeBuildRelativeUrl,
+    precomputeBuildRelativeUrl,
+  }
+}
+
+const getFilenamePattern = ({
+  ressource,
+  entryPointUrls,
+  workerUrls,
+  classicWorkerUrls,
+  serviceWorkerUrls,
+  classicServiceWorkerUrls,
+  asOriginalUrl,
+  precomputeBuildRelativeUrl,
+}) => {
+  if (ressource.isEntryPoint) {
+    const originalUrl = asOriginalUrl(ressource.url)
+    const entryPointBuildRelativeUrl = entryPointUrls[originalUrl]
+    return entryPointBuildRelativeUrl
+  }
+
+  // service worker:
+  // - MUST be at the root (same level than the HTML file)
+  //   otherwise it might be registered for the scope "/assets/" instead of "/"
+  // - MUST not be versioned
+  if (ressource.isServiceWorker) {
+    const originalUrl = asOriginalUrl(ressource.url)
+    const serviceWorkerBuildRelativeUrl = ressource.isJsModule
+      ? serviceWorkerUrls[originalUrl]
+      : classicServiceWorkerUrls[originalUrl]
+    // we could/should log a warning when service worker is not at the root
+    // we should also disable versioning for this file
+    // of the build directory
+    return serviceWorkerBuildRelativeUrl
+  }
+
+  // it's a module worker
+  if (ressource.isWorker) {
+    const originalUrl = asOriginalUrl(ressource.url)
+    const workerBuildRelativeUrl = ressource.isJsModule
+      ? workerUrls[originalUrl]
+      : classicWorkerUrls[originalUrl]
+    return workerBuildRelativeUrl
+  }
+
+  // inline ressource inherits location
+  if (ressource.isInline) {
+    // inherit parent directory location because it's an inline file
+    const importerBuildRelativeUrl = precomputeBuildRelativeUrl(
+      ressource.importer,
+    )
+    const name = urlToBasename(`file://${importerBuildRelativeUrl}`)
+    return `${name}_[hash][extname]`
+  }
+
+  // importmap.
+  // we want to force the fileName for the importmap
+  // so that we don't have to rewrite its content
+  // the goal is to put the importmap at the same relative path
+  // than in the project
+  if (ressource.contentType === "application/importmap+json") {
+    const importmapRessourceUrl = ressource.url
+    const name = urlToBasename(importmapRessourceUrl)
+    return `${name}_[hash][extname]`
+  }
+
+  if (ressource.isJsModule) {
+    // it's a js module
+    return "[name]_[hash][extname]"
+  }
+
+  return ressource.urlVersioningDisabled
+    ? "assets/[name][extname]"
+    : "assets/[name]_[hash][extname]"
+}
+
+const generateBuildRelativeUrl = (
+  fileUrl,
+  fileContent,
+  {
+    name = urlToBasename(fileUrl),
+    pattern = "[name]-[hash][extname]",
+    contentType = "application/octet-stream",
+    lineBreakNormalization = false,
+  } = {},
+) => {
+  pattern = typeof pattern === "function" ? pattern() : pattern
+  if (pattern.startsWith("./")) pattern = pattern.slice(2)
+  const buildRelativeUrl = renderFileNamePattern(pattern, {
+    dirname: () => urlToParentUrl(fileUrl),
+    name: () => name,
+    hash: () =>
+      generateContentHash(fileContent, {
+        contentType,
+        lineBreakNormalization,
+      }),
+    extname: () => urlToExtension(fileUrl),
+  })
+  return buildRelativeUrl
+}
+
+const renderFileNamePattern = (pattern, replacements) => {
+  return pattern.replace(/\[(\w+)\]/g, (_match, type) => {
+    const replacement = replacements[type]()
+    return replacement
+  })
+}
+
+// https://github.com/rollup/rollup/blob/19e50af3099c2f627451a45a84e2fa90d20246d5/src/utils/FileEmitter.ts#L47
+export const generateContentHash = (
+  stringOrBuffer,
+  {
+    contentType = "application/octet-stream",
+    lineBreakNormalization = false,
+  } = {},
+) => {
+  const hash = createHash("sha256")
+  hash.update(
+    lineBreakNormalization && contentTypeIsTextual(contentType)
+      ? normalizeLineBreaks(stringOrBuffer)
+      : stringOrBuffer,
+  )
+  return hash.digest("hex").slice(0, 8)
+}
+
+const contentTypeIsTextual = (contentType) => {
+  if (contentType.startsWith("text/")) {
+    return true
+  }
+  // catch things like application/manifest+json, application/importmap+json
+  if (/^application\/\w+\+json$/.test(contentType)) {
+    return true
+  }
+  if (CONTENT_TYPE_AS_TEXT.includes(contentType)) {
+    return true
+  }
+  return false
+}
+
+const CONTENT_TYPE_AS_TEXT = [
+  "application/javascript",
+  "application/json",
+  "image/svg+xml",
+]
+
+const normalizeLineBreaks = (stringOrBuffer) => {
+  if (typeof stringOrBuffer === "string") {
+    const stringWithLinuxBreaks = stringOrBuffer.replace(/\r\n/g, "\n")
+    return stringWithLinuxBreaks
+  }
+  return normalizeLineBreaksForBuffer(stringOrBuffer)
+}
+
+// https://github.com/nodejs/help/issues/1738#issuecomment-458460503
+const normalizeLineBreaksForBuffer = (buffer) => {
+  const int32Array = new Int32Array(buffer, 0, buffer.length)
+  const int32ArrayWithLineBreaksNormalized = int32Array.filter(
+    (element, index, typedArray) => {
+      if (element === 0x0d) {
+        if (typedArray[index + 1] === 0x0a) {
+          // Windows -> Unix
+          return false
+        }
+        // Mac OS -> Unix
+        typedArray[index] = 0x0a
+      }
+      return true
+    },
+  )
+  return Buffer.from(int32ArrayWithLineBreaksNormalized)
+}

package/src/internal/compiling/compileHtml.js

@@ -9,8 +9,8 @@ which one is being done first
 */
 
 import { createHash } from "crypto"
+
 import { require } from "../require.js"
-import { renderNamePattern } from "../renderNamePattern.js"
 
 // https://github.com/inikulin/parse5/blob/master/packages/parse5/lib/tree-adapters/default.js
 // eslint-disable-next-line import/no-unresolved
@@ -519,6 +519,13 @@ export const getUniqueNameForInlineHtmlNode = (node, nodes, pattern) => {
   })
 }
 
+const renderNamePattern = (pattern, replacements) => {
+  return pattern.replace(/\[(\w+)\]/g, (_match, type) => {
+    const replacement = replacements[type]()
+    return replacement
+  })
+}
+
 const parseHtmlAsSingleElement = (html) => {
   const parse5 = require("parse5")
   const fragment = parse5.parseFragment(html)

package/src/internal/compiling/createCompiledFileService.js

@@ -41,10 +41,10 @@ export const createCompiledFileService = ({
   outDirectoryRelativeUrl,
 
   runtimeSupport,
-  transformTopLevelAwait,
+  babelPluginMap,
   moduleOutFormat,
   importMetaFormat,
-  babelPluginMap,
+  topLevelAwait,
   groupMap,
   customCompilers,
 
@@ -185,16 +185,16 @@ export const createCompiledFileService = ({
           request,
 
           runtimeSupport,
+          babelPluginMap: babelPluginMapFromCompileId(compileId, {
+            babelPluginMap,
+            groupMap,
+          }),
           moduleOutFormat:
             moduleOutFormat === undefined
               ? compileIdModuleFormats[compileId]
               : moduleOutFormat,
           importMetaFormat,
-          transformTopLevelAwait,
-          babelPluginMap: babelPluginMapFromCompileId(compileId, {
-            babelPluginMap,
-            groupMap,
-          }),
+          topLevelAwait,
 
           sourcemapMethod,
           sourcemapExcludeSources,

package/src/internal/compiling/js-compilation-service/jsenvTransform.js

@@ -5,11 +5,9 @@ import { babelPluginTransformImportMeta } from "@jsenv/core/src/internal/babel_p
 import {
   getMinimalBabelPluginMap,
   babelPluginsFromBabelPluginMap,
-  extractSyntaxBabelPluginMap,
 } from "@jsenv/core/src/internal/compiling/babel_plugins.js"
 import { babelPluginImportMetadata } from "@jsenv/core/src/internal/compiling/babel_plugin_import_metadata.js"
 
-import { findAsyncPluginNameInBabelPluginMap } from "./findAsyncPluginNameInBabelPluginMap.js"
 import { ansiToHTML } from "./ansiToHTML.js"
 import { babelPluginRegeneratorRuntimeAsJsenvImport } from "./babel_plugin_regenerator_runtime_as_jsenv_import.js"
 import { babelPluginBabelHelpersAsJsenvImports } from "./babel_plugin_babel_helpers_as_jsenv_imports.js"
@@ -25,10 +23,9 @@ export const jsenvTransform = async ({
   babelPluginMap,
   moduleOutFormat,
   importMetaFormat = moduleOutFormat,
+  topLevelAwait,
 
   babelHelpersInjectionAsImport,
-  allowTopLevelAwait,
-  transformTopLevelAwait,
   transformGenerator,
   regeneratorRuntimeImportPath,
   sourcemapEnabled,
@@ -50,7 +47,11 @@ export const jsenvTransform = async ({
     sourceFileName: inputPath,
     // https://babeljs.io/docs/en/options#parseropts
     parserOpts: {
-      allowAwaitOutsideFunction: allowTopLevelAwait,
+      allowAwaitOutsideFunction:
+        topLevelAwait === undefined ||
+        topLevelAwait === "return" ||
+        topLevelAwait === "simple" ||
+        topLevelAwait === "ignore",
     },
     generatorOpts: {
       compact: false,
@@ -147,56 +148,9 @@ export const jsenvTransform = async ({
     "import-metadata": [babelPluginImportMetadata],
   }
 
-  const asyncPluginName = findAsyncPluginNameInBabelPluginMap(babelPluginMap)
-
-  if (
-    moduleOutFormat === "systemjs" &&
-    transformTopLevelAwait &&
-    asyncPluginName
-  ) {
-    const babelPluginMapWithoutAsync = {
-      ...babelPluginMap,
-      "proposal-dynamic-import": [proposalDynamicImport],
-      "transform-modules-systemjs": [transformModulesSystemJs],
-    }
-    delete babelPluginMapWithoutAsync[asyncPluginName]
-
-    // put body inside something like (async () => {})()
-    const result = await babelTransform({
-      ast,
-      code,
-      options: {
-        ...options,
-        plugins: babelPluginsFromBabelPluginMap(babelPluginMapWithoutAsync),
-      },
-    })
-
-    // we need to retranspile the await keywords now wrapped
-    // inside Systemjs function.
-    // They are ignored, at least by transform-async-to-promises
-    // see https://github.com/rpetrich/babel-plugin-transform-async-to-promises/issues/26
-    const { babelSyntaxPluginMap } = extractSyntaxBabelPluginMap(babelPluginMap)
-    const finalResult = await babelTransform({
-      // ast: result.ast,
-      code: result.code,
-      options: {
-        ...options,
-        // about inputSourceMap see
-        // https://github.com/babel/babel/blob/eac4c5bc17133c2857f2c94c1a6a8643e3b547a7/packages/babel-core/src/transformation/file/generate.js#L57
-        // https://github.com/babel/babel/blob/090c364a90fe73d36a30707fc612ce037bdbbb24/packages/babel-core/src/transformation/file/merge-map.js#L6s
-        inputSourceMap: result.map,
-        plugins: babelPluginsFromBabelPluginMap({
-          ...babelSyntaxPluginMap,
-          [asyncPluginName]: babelPluginMap[asyncPluginName],
-        }),
-      },
-    })
-
-    return {
-      ...result,
-      ...finalResult,
-      metadata: { ...result.metadata, ...finalResult.metadata },
-    }
+  const asyncToPromise = babelPluginMap["transform-async-to-promises"]
+  if (topLevelAwait && asyncToPromise) {
+    asyncToPromise.options.topLevelAwait = topLevelAwait
   }
 
   const babelTransformReturnValue = await babelTransform({

package/src/internal/compiling/js-compilation-service/transformJs.js

@@ -11,9 +11,8 @@ export const transformJs = async ({
   babelPluginMap,
   moduleOutFormat = "esmodule",
   importMetaFormat = moduleOutFormat,
-  babelHelpersInjectionAsImport = true,
-  allowTopLevelAwait = true,
-  transformTopLevelAwait = true,
+  babelHelpersInjectionAsImport = moduleOutFormat === "esmodule",
+  topLevelAwait,
   transformGenerator = true,
   sourcemapEnabled = true,
 }) => {
@@ -33,6 +32,11 @@ export const transformJs = async ({
   if (typeof url !== "string") {
     throw new TypeError(`url must be a string, got ${url}`)
   }
+  if (babelHelpersInjectionAsImport && moduleOutFormat !== "esmodule") {
+    throw new Error(
+      `babelHelpersInjectionAsImport can be enabled only when "moduleOutFormat" is "esmodule"`,
+    )
+  }
 
   const transformResult = await jsenvTransform({
     code,
@@ -47,8 +51,7 @@ export const transformJs = async ({
     importMetaFormat,
 
     babelHelpersInjectionAsImport,
-    allowTopLevelAwait,
-    transformTopLevelAwait,
+    topLevelAwait,
     transformGenerator,
     sourcemapEnabled,
   })

package/src/internal/compiling/jsenvCompilerForHtml.js

@@ -42,10 +42,10 @@ export const compileHtml = async ({
   outDirectoryRelativeUrl,
   compileId,
 
-  transformTopLevelAwait,
+  babelPluginMap,
   moduleOutFormat,
   importMetaFormat,
-  babelPluginMap,
+  topLevelAwait,
 
   sourcemapMethod,
 
@@ -284,10 +284,10 @@ export const compileHtml = async ({
           compiledUrl: scriptCompiledFileUrl,
           projectDirectoryUrl,
 
-          transformTopLevelAwait,
+          babelPluginMap,
           moduleOutFormat,
           importMetaFormat,
-          babelPluginMap,
+          topLevelAwait,
         })
       } catch (e) {
         // If there is a syntax error in inline script

package/src/internal/compiling/jsenvCompilerForJavaScript.js

@@ -9,9 +9,9 @@ export const compileJavascript = async ({
   projectDirectoryUrl,
 
   babelPluginMap,
-  transformTopLevelAwait,
   moduleOutFormat,
   importMetaFormat,
+  topLevelAwait,
 
   sourcemapExcludeSources,
   sourcemapMethod,
@@ -24,9 +24,9 @@ export const compileJavascript = async ({
     projectDirectoryUrl,
 
     babelPluginMap,
-    transformTopLevelAwait,
     moduleOutFormat,
     importMetaFormat,
+    topLevelAwait,
   })
 
   return transformResultToCompilationResult(

package/src/internal/compiling/startCompileServer.js

@@ -82,9 +82,9 @@ export const startCompileServer = async ({
   projectFileCacheStrategy = "mtime",
 
   // js compile options
-  transformTopLevelAwait = true,
   moduleOutFormat,
   importMetaFormat,
+  topLevelAwait,
   env = {},
   processEnvNodeEnv = process.env.NODE_ENV,
   replaceProcessEnvNodeEnv = true,
@@ -310,7 +310,7 @@ export const startCompileServer = async ({
       importDefaultExtension,
 
       runtimeSupport,
-      transformTopLevelAwait,
+      topLevelAwait,
       groupMap: compileServerGroupMap,
       babelPluginMap,
       customCompilers,