@jsenv/core 24.6.3 → 25.0.0-alpha.1

package/src/internal/building/url_loader.js

@@ -1,17 +1,12 @@
-import { resolveUrl, urlToFilename } from "@jsenv/filesystem"
-
-import { transformJs } from "@jsenv/core/src/internal/compiling/js-compilation-service/transformJs.js"
-import { convertCssTextToJavascriptModule } from "@jsenv/core/src/internal/building/css_module.js"
+import { convertJsonTextToJavascriptModule } from "@jsenv/core/src/internal/building/json_module.js"
 import { getJavaScriptSourceMappingUrl } from "@jsenv/core/src/internal/sourceMappingURLUtils.js"
+
 import { loadSourcemap } from "./sourcemap_loader.js"
 
 export const createUrlLoader = ({
-  projectDirectoryUrl,
-  buildDirectoryUrl,
-  babelPluginMap,
+  urlCustomLoaders,
   allowJson,
   urlImporterMap,
-  inlineModuleScripts,
 
   asServerUrl,
   asProjectUrl,
@@ -21,112 +16,13 @@ export const createUrlLoader = ({
 }) => {
   const urlResponseBodyMap = {}
 
-  const loadUrl = async (rollupUrl, { signal, logger, ressourceBuilder }) => {
+  const loadUrl = async (rollupUrl, { signal, logger }) => {
     let url = asServerUrl(rollupUrl)
-    const { importType, urlWithoutImportType } = extractImportTypeFromUrl(url)
-    // importing CSS from JS with import assertions
-    if (importType === "css") {
-      const importer = urlImporterMap[url]
-      const cssReference =
-        await ressourceBuilder.createReferenceFoundInJsModule({
-          referenceLabel: "css import assertion",
-          // If all references to a ressource are only import assertions
-          // the file referenced do not need to be written on filesystem
-          // as it was converted to a js file
-          // We pass "isImportAssertion: true" for this purpose
-          isImportAssertion: true,
-          jsUrl: importer.url,
-          jsLine: importer.line,
-          jsColumn: importer.column,
-          ressourceSpecifier: urlWithoutImportType,
-          contentTypeExpected: "text/css",
-        })
-      await cssReference.ressource.getReadyPromise()
-      const cssBuildUrl = resolveUrl(
-        cssReference.ressource.buildRelativeUrl,
-        buildDirectoryUrl,
-      )
-      const jsBuildUrl = resolveUrl(
-        urlToFilename(importer.url),
-        buildDirectoryUrl,
-      )
-      let code = String(cssReference.ressource.bufferAfterBuild)
-      let map
-      const sourcemapReference = cssReference.ressource.dependencies.find(
-        (dependency) => {
-          return dependency.ressource.isSourcemap
-        },
-      )
-      if (sourcemapReference) {
-        // because css is ready, it's sourcemap is also ready
-        // we can read directly sourcemapReference.ressource.bufferAfterBuild
-        map = JSON.parse(sourcemapReference.ressource.bufferAfterBuild)
-      }
-
-      const jsModuleConversionResult = await convertCssTextToJavascriptModule({
-        cssUrl: cssBuildUrl,
-        jsUrl: jsBuildUrl,
-        code,
-        map,
-      })
-      code = jsModuleConversionResult.code
-      map = jsModuleConversionResult.map
-
-      return {
-        url,
-        code,
-        map,
-      }
-    }
-    // importing json from JS with import assertion
-    if (importType === "json") {
-      const importer = urlImporterMap[url]
-      const jsonReference =
-        await ressourceBuilder.createReferenceFoundInJsModule({
-          referenceLabel: "json import assertion",
-          // If all references to a ressource are only import assertions
-          // the file referenced do not need to be written on filesystem
-          // as it was converted to a js file
-          // We pass "isImportAssertion: true" for this purpose
-          isImportAssertion: true,
-          jsUrl: importer.url,
-          jsLine: importer.line,
-          jsColumn: importer.column,
-          ressourceSpecifier: asServerUrl(urlWithoutImportType),
-          contentTypeExpected: "application/json",
-        })
-      await jsonReference.ressource.getReadyPromise()
-      let code = String(jsonReference.ressource.bufferAfterBuild)
-      let map
-
-      const jsModuleConversionResult = await convertJsonTextToJavascriptModule({
-        code,
-        map,
-      })
-      code = jsModuleConversionResult.code
-      map = jsModuleConversionResult.map
 
-      return {
-        url,
-        code,
-      }
-    }
-
-    if (url in inlineModuleScripts) {
-      const transformResult = await transformJs({
-        code: String(inlineModuleScripts[url].bufferBeforeBuild),
-        url: asOriginalUrl(url), // transformJs expect a file:// url
-        projectDirectoryUrl,
-        babelPluginMap,
-        // moduleOutFormat: format // we are compiling for rollup output must be "esmodule"
-      })
-      let code = transformResult.code
-      let map = transformResult.map
-      return {
-        url,
-        code,
-        map,
-      }
+    const customLoader = urlCustomLoaders[url]
+    if (customLoader) {
+      const result = await customLoader()
+      return result
     }
 
     const response = await urlFetcher.fetchUrl(url, {
@@ -233,33 +129,3 @@ export const createUrlLoader = ({
     getUrlResponseBodyMap: () => urlResponseBodyMap,
   }
 }
-
-const extractImportTypeFromUrl = (url) => {
-  const urlObject = new URL(url)
-  const { search } = urlObject
-  const searchParams = new URLSearchParams(search)
-
-  const importType = searchParams.get("import_type")
-  if (!importType) {
-    return {}
-  }
-
-  searchParams.delete("import_type")
-  urlObject.search = String(searchParams)
-  return {
-    importType,
-    urlWithoutImportType: urlObject.href,
-  }
-}
-
-const convertJsonTextToJavascriptModule = ({ code }) => {
-  // here we could do the following
-  // return export default jsonText
-  // This would return valid js, that would be minified later
-  // however we will prefer using JSON.parse because it's faster
-  // for js engine to parse JSON than JS
-
-  return {
-    code: `export default JSON.parse(${JSON.stringify(code.trim())})`,
-  }
-}

package/src/internal/building/url_versioning.js

@@ -0,0 +1,220 @@
+import { createHash } from "crypto"
+import {
+  urlToParentUrl,
+  urlToBasename,
+  urlToExtension,
+} from "@jsenv/filesystem"
+
+export const createUrlVersioner = ({
+  entryPointUrls,
+  asOriginalUrl,
+  lineBreakNormalization,
+}) => {
+  const names = []
+  const getFreeName = (name) => {
+    let nameCandidate = name
+    let integer = 1
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      if (!names.includes(nameCandidate)) {
+        names.push(nameCandidate)
+        return nameCandidate
+      }
+      integer++
+      nameCandidate = `${name}${integer}`
+    }
+  }
+
+  const computeBuildRelativeUrl = (ressource, precomputation) => {
+    const pattern = getFilenamePattern({
+      ressource,
+      entryPointUrls,
+      asOriginalUrl,
+      precomputeBuildRelativeUrl,
+    })
+
+    const buildRelativeUrl = generateBuildRelativeUrl(
+      ressource.url,
+      ressource.bufferAfterBuild,
+      {
+        pattern,
+        getName: precomputation
+          ? () => urlToBasename(ressource.url)
+          : () => getFreeName(urlToBasename(ressource.url)),
+        contentType: ressource.contentType,
+        lineBreakNormalization,
+      },
+    )
+    return buildRelativeUrl
+  }
+
+  const precomputeBuildRelativeUrl = (ressource, bufferAfterBuild = "") => {
+    if (ressource.buildRelativeUrl) {
+      return ressource.buildRelativeUrl
+    }
+    if (ressource.precomputedBuildRelativeUrl) {
+      return ressource.precomputedBuildRelativeUrl
+    }
+
+    ressource.bufferAfterBuild = bufferAfterBuild
+    const precomputedBuildRelativeUrl = computeBuildRelativeUrl(ressource, true)
+    ressource.bufferAfterBuild = undefined
+    ressource.precomputedBuildRelativeUrl = precomputedBuildRelativeUrl
+    return precomputedBuildRelativeUrl
+  }
+
+  return {
+    computeBuildRelativeUrl,
+    precomputeBuildRelativeUrl,
+  }
+}
+
+const getFilenamePattern = ({
+  ressource,
+  entryPointUrls,
+  asOriginalUrl,
+  precomputeBuildRelativeUrl,
+}) => {
+  if (ressource.isEntryPoint) {
+    const originalUrl = asOriginalUrl(ressource.url)
+    const entryPointBuildRelativeUrl = entryPointUrls[originalUrl]
+    return entryPointBuildRelativeUrl
+  }
+
+  // inline ressource inherits location
+  if (ressource.isInline) {
+    // inherit parent directory location because it's an inline file
+    const importerBuildRelativeUrl = precomputeBuildRelativeUrl(
+      ressource.importer,
+    )
+    const name = urlToBasename(`file://${importerBuildRelativeUrl}`)
+    return `${name}_[hash][extname]`
+  }
+
+  // importmap.
+  // we want to force the fileName for the importmap
+  // so that we don't have to rewrite its content
+  // the goal is to put the importmap at the same relative path
+  // than in the project
+  if (ressource.contentType === "application/importmap+json") {
+    const importmapRessourceUrl = ressource.url
+    const name = urlToBasename(importmapRessourceUrl)
+    return `${name}_[hash][extname]`
+  }
+
+  // service worker:
+  // - MUST be at the root (same level than the HTML file)
+  //   otherwise it might be registered for the scope "/assets/" instead of "/"
+  // - MUST not be versioned
+  if (ressource.isServiceWorker) {
+    return "[name][extname]"
+  }
+
+  if (ressource.isWorker) {
+    return "[name]_[hash][extname]"
+  }
+
+  if (ressource.isJsModule) {
+    return "[name]_[hash][extname]"
+  }
+
+  return ressource.urlVersioningDisabled
+    ? "assets/[name][extname]"
+    : "assets/[name]_[hash][extname]"
+}
+
+const generateBuildRelativeUrl = (
+  fileUrl,
+  fileContent,
+  {
+    getName,
+    pattern,
+    contentType = "application/octet-stream",
+    lineBreakNormalization = false,
+  } = {},
+) => {
+  pattern = typeof pattern === "function" ? pattern() : pattern
+  if (pattern.startsWith("./")) pattern = pattern.slice(2)
+  const buildRelativeUrl = renderFileNamePattern(pattern, {
+    dirname: () => urlToParentUrl(fileUrl),
+    name: getName,
+    hash: () =>
+      generateContentHash(fileContent, {
+        contentType,
+        lineBreakNormalization,
+      }),
+    extname: () => urlToExtension(fileUrl),
+  })
+  return buildRelativeUrl
+}
+
+const renderFileNamePattern = (pattern, replacements) => {
+  return pattern.replace(/\[(\w+)\]/g, (_match, type) => {
+    const replacement = replacements[type]()
+    return replacement
+  })
+}
+
+// https://github.com/rollup/rollup/blob/19e50af3099c2f627451a45a84e2fa90d20246d5/src/utils/FileEmitter.ts#L47
+export const generateContentHash = (
+  stringOrBuffer,
+  {
+    contentType = "application/octet-stream",
+    lineBreakNormalization = false,
+  } = {},
+) => {
+  const hash = createHash("sha256")
+  hash.update(
+    lineBreakNormalization && contentTypeIsTextual(contentType)
+      ? normalizeLineBreaks(stringOrBuffer)
+      : stringOrBuffer,
+  )
+  return hash.digest("hex").slice(0, 8)
+}
+
+const contentTypeIsTextual = (contentType) => {
+  if (contentType.startsWith("text/")) {
+    return true
+  }
+  // catch things like application/manifest+json, application/importmap+json
+  if (/^application\/\w+\+json$/.test(contentType)) {
+    return true
+  }
+  if (CONTENT_TYPE_AS_TEXT.includes(contentType)) {
+    return true
+  }
+  return false
+}
+
+const CONTENT_TYPE_AS_TEXT = [
+  "application/javascript",
+  "application/json",
+  "image/svg+xml",
+]
+
+const normalizeLineBreaks = (stringOrBuffer) => {
+  if (typeof stringOrBuffer === "string") {
+    const stringWithLinuxBreaks = stringOrBuffer.replace(/\r\n/g, "\n")
+    return stringWithLinuxBreaks
+  }
+  return normalizeLineBreaksForBuffer(stringOrBuffer)
+}
+
+// https://github.com/nodejs/help/issues/1738#issuecomment-458460503
+const normalizeLineBreaksForBuffer = (buffer) => {
+  const int32Array = new Int32Array(buffer, 0, buffer.length)
+  const int32ArrayWithLineBreaksNormalized = int32Array.filter(
+    (element, index, typedArray) => {
+      if (element === 0x0d) {
+        if (typedArray[index + 1] === 0x0a) {
+          // Windows -> Unix
+          return false
+        }
+        // Mac OS -> Unix
+        typedArray[index] = 0x0a
+      }
+      return true
+    },
+  )
+  return Buffer.from(int32ArrayWithLineBreaksNormalized)
+}

package/src/internal/compiling/compileHtml.js

@@ -9,8 +9,8 @@ which one is being done first
 */
 
 import { createHash } from "crypto"
+
 import { require } from "../require.js"
-import { renderNamePattern } from "../renderNamePattern.js"
 
 // https://github.com/inikulin/parse5/blob/master/packages/parse5/lib/tree-adapters/default.js
 // eslint-disable-next-line import/no-unresolved
@@ -519,6 +519,13 @@ export const getUniqueNameForInlineHtmlNode = (node, nodes, pattern) => {
   })
 }
 
+const renderNamePattern = (pattern, replacements) => {
+  return pattern.replace(/\[(\w+)\]/g, (_match, type) => {
+    const replacement = replacements[type]()
+    return replacement
+  })
+}
+
 const parseHtmlAsSingleElement = (html) => {
   const parse5 = require("parse5")
   const fragment = parse5.parseFragment(html)

package/src/internal/compiling/createCompiledFileService.js

@@ -6,10 +6,10 @@ import {
   urlToMeta,
 } from "@jsenv/filesystem"
 
-import { featuresCompatMap } from "@jsenv/core/src/internal/generateGroupMap/featuresCompatMap.js"
 import { createRuntimeCompat } from "@jsenv/core/src/internal/generateGroupMap/runtime_compat.js"
-
+import { shakeBabelPluginMap } from "@jsenv/core/src/internal/generateGroupMap/shake_babel_plugin_map.js"
 import { serverUrlToCompileInfo } from "@jsenv/core/src/internal/url_conversion.js"
+
 import { setUrlExtension } from "../url_utils.js"
 import {
   COMPILE_ID_BUILD_GLOBAL,
@@ -41,12 +41,16 @@ export const createCompiledFileService = ({
   outDirectoryRelativeUrl,
 
   runtimeSupport,
-  transformTopLevelAwait,
+  babelPluginMap,
   moduleOutFormat,
   importMetaFormat,
-  babelPluginMap,
+  topLevelAwait,
+  prependSystemJs,
   groupMap,
   customCompilers,
+  workerUrls,
+  serviceWorkerUrls,
+  importMapInWebWorkers,
 
   jsenvEventSourceClientInjection,
   jsenvToolbarInjection,
@@ -60,6 +64,8 @@ export const createCompiledFileService = ({
   Object.keys(groupMap).forEach((groupName) => {
     compileIdModuleFormats[groupName] = canAvoidSystemJs({
       runtimeSupport: groupMap[groupName].minRuntimeVersions,
+      workerUrls,
+      importMapInWebWorkers,
     })
       ? "esmodule"
       : "systemjs"
@@ -183,18 +189,20 @@ export const createCompiledFileService = ({
           outDirectoryRelativeUrl,
           compileId,
           request,
-
+          babelPluginMap: shakeBabelPluginMap({
+            babelPluginMap,
+            missingFeatureNames: groupMap[compileId].missingFeatureNames,
+          }),
           runtimeSupport,
+          workerUrls,
+          serviceWorkerUrls,
           moduleOutFormat:
             moduleOutFormat === undefined
               ? compileIdModuleFormats[compileId]
               : moduleOutFormat,
           importMetaFormat,
-          transformTopLevelAwait,
-          babelPluginMap: babelPluginMapFromCompileId(compileId, {
-            babelPluginMap,
-            groupMap,
-          }),
+          topLevelAwait,
+          prependSystemJs,
 
           sourcemapMethod,
           sourcemapExcludeSources,
@@ -210,18 +218,23 @@ export const createCompiledFileService = ({
   }
 }
 
-const canAvoidSystemJs = ({ runtimeSupport }) => {
+const canAvoidSystemJs = ({
+  runtimeSupport,
+  workerUrls,
+  importMapInWebWorkers,
+}) => {
   const runtimeCompatMap = createRuntimeCompat({
+    featureNames: [
+      "module",
+      "importmap",
+      "import_assertion_type_json",
+      "import_assertion_type_css",
+      ...(workerUrls.length > 0 ? ["worker_type_module"] : []),
+      ...(importMapInWebWorkers ? ["worker_importmap"] : []),
+    ],
     runtimeSupport,
-    pluginMap: {
-      module: true,
-      importmap: true,
-      import_assertion_type_json: true,
-      import_assertion_type_css: true,
-    },
-    pluginCompatMap: featuresCompatMap,
   })
-  return runtimeCompatMap.pluginRequiredNameArray.length === 0
+  return runtimeCompatMap.missingFeatureNames.length === 0
 }
 
 const getCompiler = ({ originalFileUrl, compileMeta }) => {
@@ -295,31 +308,6 @@ const contentTypeExtensions = {
   // "text/css": ".css",
 }
 
-const babelPluginMapFromCompileId = (
-  compileId,
-  { babelPluginMap, groupMap },
-) => {
-  const babelPluginMapForGroup = {}
-
-  groupMap[compileId].pluginRequiredNameArray.forEach((requiredPluginName) => {
-    const babelPlugin = babelPluginMap[requiredPluginName]
-    if (babelPlugin) {
-      babelPluginMapForGroup[requiredPluginName] = babelPlugin
-    }
-  })
-
-  Object.keys(babelPluginMap).forEach((key) => {
-    if (key.startsWith("syntax-")) {
-      babelPluginMapForGroup[key] = babelPluginMap[key]
-    }
-    if (key === "transform-replace-expressions") {
-      babelPluginMapForGroup[key] = babelPluginMap[key]
-    }
-  })
-
-  return babelPluginMapForGroup
-}
-
 const ressourceToPathname = (ressource) => {
   const searchSeparatorIndex = ressource.indexOf("?")
   const pathname =