@jsenv/core 25.1.1 → 25.3.0

package/src/internal/compiling/jsenvCompilerForJavaScript.js

@@ -1,12 +1,13 @@
+import { generateSourcemapUrl } from "@jsenv/core/src/internal/sourceMappingURLUtils.js"
+
 import { transformJs } from "./js-compilation-service/transformJs.js"
 import { transformResultToCompilationResult } from "./transformResultToCompilationResult.js"
 
 export const compileJavascript = async ({
-  code,
-  map,
+  projectDirectoryUrl,
+  jsenvRemoteDirectory,
   url,
   compiledUrl,
-  projectDirectoryUrl,
 
   babelPluginMap,
   workerUrls,
@@ -16,6 +17,8 @@ export const compileJavascript = async ({
   topLevelAwait,
   prependSystemJs,
 
+  code,
+  map,
   sourcemapExcludeSources,
   sourcemapMethod,
 }) => {
@@ -23,38 +26,39 @@ export const compileJavascript = async ({
     prependSystemJs =
       workerUrls.includes(url) || serviceWorkerUrls.includes(url)
   }
-
   const transformResult = await transformJs({
-    code,
-    map,
+    projectDirectoryUrl,
+    jsenvRemoteDirectory,
     url,
     compiledUrl,
-    projectDirectoryUrl,
 
     babelPluginMap,
     moduleOutFormat,
     importMetaFormat,
     topLevelAwait,
     prependSystemJs,
-  })
 
+    code,
+    map,
+  })
   return transformResultToCompilationResult(
     {
       contentType: "application/javascript",
+      metadata: transformResult.metadata,
       code: transformResult.code,
       map: transformResult.map,
-      metadata: transformResult.metadata,
     },
     {
       projectDirectoryUrl,
-      originalFileContent: code,
+      jsenvRemoteDirectory,
       originalFileUrl: url,
       compiledFileUrl: compiledUrl,
       // sourcemap are not inside the asset folder because
       // of https://github.com/microsoft/vscode-chrome-debug-core/issues/544
-      sourcemapFileUrl: `${compiledUrl}.map`,
+      sourcemapFileUrl: generateSourcemapUrl(compiledUrl),
       sourcemapExcludeSources,
       sourcemapMethod,
+      originalFileContent: code,
     },
   )
 }

package/src/internal/compiling/startCompileServer.js

@@ -36,17 +36,19 @@ import {
 } from "@jsenv/core/dist/build_manifest.js"
 import { generateGroupMap } from "@jsenv/core/src/internal/generateGroupMap/generateGroupMap.js"
 import { isBrowserPartOfSupportedRuntimes } from "@jsenv/core/src/internal/generateGroupMap/runtime_support.js"
-import { loadBabelPluginMapFromFile } from "./load_babel_plugin_map_from_file.js"
-import { extractSyntaxBabelPluginMap } from "./babel_plugins.js"
+
+import { createJsenvRemoteDirectory } from "../jsenv_remote_directory.js"
 import {
   sourcemapMainFileInfo,
   sourcemapMappingFileInfo,
 } from "../jsenvInternalFiles.js"
+import { babelPluginReplaceExpressions } from "../babel_plugin_replace_expressions.js"
 import {
   jsenvCoreDirectoryUrl,
   jsenvDistDirectoryUrl,
 } from "../jsenvCoreDirectoryUrl.js"
-import { babelPluginReplaceExpressions } from "../babel_plugin_replace_expressions.js"
+import { loadBabelPluginMapFromFile } from "./load_babel_plugin_map_from_file.js"
+import { extractSyntaxBabelPluginMap } from "./babel_plugins.js"
 import { babelPluginGlobalThisAsJsenvImport } from "./babel_plugin_global_this_as_jsenv_import.js"
 import { babelPluginNewStylesheetAsJsenvImport } from "./babel_plugin_new_stylesheet_as_jsenv_import.js"
 import { babelPluginImportAssertions } from "./babel_plugin_import_assertions.js"
@@ -98,6 +100,7 @@ export const startCompileServer = async ({
   babelPluginMap,
   babelConfigFileUrl,
   customCompilers = {},
+  preservedUrls,
   workers = [],
   serviceWorkers = [],
   importMapInWebWorkers = false,
@@ -146,6 +149,30 @@ export const startCompileServer = async ({
   )
   const logger = createLogger({ logLevel })
 
+  preservedUrls = {
+    // Authorize jsenv to modify any file url
+    // because the goal is to build the files into chunks
+    "file://": false,
+    // Preserves http and https urls
+    // because if code specifiy a CDN url it's usually because code wants
+    // to keep the url intact and keep HTTP request to CDN (both in dev and prod)
+    "http://": true,
+    "https://": true,
+    /*
+     * It's possible to selectively overrides the behaviour above:
+     * 1. The CDN file needs to be transformed to be executable in dev, build or both
+     * preservedUrls: {"https://cdn.skypack.dev/preact@10.6.4": false}
+     * 2. No strong need to preserve the CDN dependency
+     * 3. Prevent concatenation of a file during build
+     * preservedUrls: {"./file.js": false}
+     */
+    ...preservedUrls,
+  }
+  const jsenvRemoteDirectory = createJsenvRemoteDirectory({
+    projectDirectoryUrl,
+    jsenvDirectoryRelativeUrl,
+    preservedUrls,
+  })
   const workerUrls = workers.map((worker) =>
     resolveUrl(worker, projectDirectoryUrl),
   )
@@ -290,11 +317,18 @@ export const startCompileServer = async ({
     projectDirectoryUrl,
     jsenvDirectoryRelativeUrl,
     outDirectoryRelativeUrl,
+    jsenvRemoteDirectory,
     importDefaultExtension,
+
+    preservedUrls,
+    workers,
+    serviceWorkers,
     compileServerGroupMap,
+    babelPluginMap,
+    replaceProcessEnvNodeEnv,
+    processEnvNodeEnv,
     env,
     inlineImportMapIntoHTML,
-    babelPluginMap,
     customCompilers,
     jsenvToolbarInjection,
     sourcemapMethod,
@@ -328,7 +362,9 @@ export const startCompileServer = async ({
       logger,
 
       projectDirectoryUrl,
+      jsenvDirectoryRelativeUrl,
       outDirectoryRelativeUrl,
+      jsenvRemoteDirectory,
 
       importDefaultExtension,
 
@@ -369,6 +405,7 @@ export const startCompileServer = async ({
       : {}),
     "service:source file": createSourceFileService({
       projectDirectoryUrl,
+      jsenvRemoteDirectory,
       projectFileRequestedCallback,
       projectFileCacheStrategy,
     }),
@@ -424,6 +461,7 @@ export const startCompileServer = async ({
     ...compileServer,
     compileServerGroupMap,
     babelPluginMap,
+    preservedUrls,
     projectFileRequestedCallback,
   }
 }
@@ -939,30 +977,45 @@ const createSourceFileService = ({
   projectDirectoryUrl,
   projectFileRequestedCallback,
   projectFileCacheStrategy,
+  jsenvRemoteDirectory,
 }) => {
   return async (request) => {
     const relativeUrl = request.pathname.slice(1)
     projectFileRequestedCallback(relativeUrl, request)
-
     const fileUrl = new URL(request.ressource.slice(1), projectDirectoryUrl)
       .href
     const fileIsInsideJsenvDistDirectory = urlIsInsideOf(
       fileUrl,
       jsenvDistDirectoryUrl,
     )
-
-    const responsePromise = fetchFileSystem(fileUrl, {
-      headers: request.headers,
-      etagEnabled: projectFileCacheStrategy === "etag",
-      mtimeEnabled: projectFileCacheStrategy === "mtime",
-      ...(fileIsInsideJsenvDistDirectory
-        ? {
-            cacheControl: `private,max-age=${60 * 60 * 24 * 30},immutable`,
-          }
-        : {}),
-    })
-
-    return responsePromise
+    const fromFileSystem = () =>
+      fetchFileSystem(fileUrl, {
+        headers: request.headers,
+        etagEnabled: projectFileCacheStrategy === "etag",
+        mtimeEnabled: projectFileCacheStrategy === "mtime",
+        ...(fileIsInsideJsenvDistDirectory
+          ? {
+              cacheControl: `private,max-age=${60 * 60 * 24 * 30},immutable`,
+            }
+          : {}),
+      })
+    const filesystemResponse = await fromFileSystem()
+    if (
+      filesystemResponse.status === 404 &&
+      jsenvRemoteDirectory.isFileUrlForRemoteUrl(fileUrl)
+    ) {
+      try {
+        await jsenvRemoteDirectory.loadFileUrlFromRemote(fileUrl, request)
+        // re-fetch filesystem instead to ensure response headers are correct
+        return fromFileSystem()
+      } catch (e) {
+        if (e && e.asResponse) {
+          return e.asResponse()
+        }
+        throw e
+      }
+    }
+    return filesystemResponse
   }
 }
 
@@ -971,6 +1024,10 @@ const createCompileServerMetaFileInfo = ({
   jsenvDirectoryRelativeUrl,
   outDirectoryRelativeUrl,
   importDefaultExtension,
+
+  preservedUrls,
+  workers,
+  serviceWorkers,
   compileServerGroupMap,
   babelPluginMap,
   replaceProcessEnvNodeEnv,
@@ -1016,6 +1073,9 @@ const createCompileServerMetaFileInfo = ({
     outDirectoryRelativeUrl,
     importDefaultExtension,
 
+    preservedUrls,
+    workers,
+    serviceWorkers,
     babelPluginMap: babelPluginMapAsData(babelPluginMap),
     compileServerGroupMap,
     customCompilerPatterns,
@@ -1029,7 +1089,7 @@ const createCompileServerMetaFileInfo = ({
     sourcemapMappingFileRelativeUrl,
     errorStackRemapping: true,
 
-    // used to consider the logic generating files may have changed
+    // used to consider logic generating files may have changed
     jsenvCorePackageVersion,
 
     // impact only HTML files

package/src/internal/compiling/transformResultToCompilationResult.js

@@ -3,6 +3,7 @@ import {
   urlToRelativeUrl,
   readFile,
   ensureWindowsDriveLetter,
+  urlIsInsideOf,
 } from "@jsenv/filesystem"
 
 import {
@@ -15,16 +16,16 @@ import {
   setCssSourceMappingUrl,
   sourcemapToBase64Url,
 } from "../sourceMappingURLUtils.js"
-import { generateCompiledFileAssetUrl } from "./compile-directory/compile-asset.js"
+import { generateCompilationAssetUrl } from "./compile-directory/compile-asset.js"
 import { testFilePresence } from "./compile-directory/fs-optimized-for-cache.js"
 
 const isWindows = process.platform === "win32"
 
 export const transformResultToCompilationResult = async (
-  { contentType, code, map, metadata = {} },
+  { contentType, metadata = {}, code, map },
   {
     projectDirectoryUrl,
-    originalFileContent,
+    jsenvRemoteDirectory,
     originalFileUrl,
     compiledFileUrl,
     sourcemapFileUrl,
@@ -33,9 +34,13 @@ export const transformResultToCompilationResult = async (
     // it also means client have to fetch source from server (additional http request)
     // some client ignore sourcesContent property such as vscode-chrome-debugger
     // Because it's the most complex scenario and we want to ensure client is always able
-    // to find source from the sourcemap, we remove map.sourcesContent by default to test this.
-    sourcemapExcludeSources = true,
+    // to find source from the sourcemap, it's a good idea
+    // to exclude sourcesContent from sourcemap.
+    // However some ressource are abstract and it means additional http request for the browser.
+    // For these reasons it's simpler to keep source content in sourcemap.
+    sourcemapExcludeSources = false,
     sourcemapMethod = "comment", // "comment", "inline"
+    originalFileContent,
   },
 ) => {
   if (typeof contentType !== "string") {
@@ -71,6 +76,14 @@ export const transformResultToCompilationResult = async (
   const sourcesContent = []
   const assets = []
   const assetsContent = []
+  const addSource = ({ url, content }) => {
+    sources.push(url)
+    sourcesContent.push(content)
+  }
+  const addAsset = ({ url, content }) => {
+    assets.push(url)
+    assetsContent.push(content)
+  }
 
   let output = code
   if (sourcemapEnabled && map) {
@@ -78,8 +91,10 @@ export const transformResultToCompilationResult = async (
       // may happen in some cases where babel returns a wrong sourcemap
       // there is at least one case where it happens
       // a file with only import './whatever.js' inside
-      sources.push(originalFileUrl)
-      sourcesContent.push(originalFileContent)
+      addSource({
+        url: originalFileUrl,
+        content: originalFileContent,
+      })
     } else {
       map.sources.forEach((source, index) => {
         const sourceFileUrl = resolveSourceFile({
@@ -90,19 +105,25 @@ export const transformResultToCompilationResult = async (
           projectDirectoryUrl,
         })
         if (sourceFileUrl) {
-          map.sources[index] = urlToRelativeUrl(sourceFileUrl, sourcemapFileUrl)
+          // In case the file comes from a remote url
+          // we prefer to consider remote url as the real source for this code
+          map.sources[index] =
+            jsenvRemoteDirectory &&
+            jsenvRemoteDirectory.isFileUrlForRemoteUrl(sourceFileUrl)
+              ? jsenvRemoteDirectory.remoteUrlFromFileUrl(sourceFileUrl)
+              : urlToRelativeUrl(sourceFileUrl, sourcemapFileUrl)
           sources[index] = sourceFileUrl
         }
       })
-
       if (sources.length === 0) {
         // happens when sourcemap is generated by webpack and looks like
         // webpack://Package./src/file.js
         // in that case we'll don't know how to find the source file
-        sources.push(originalFileUrl)
-        sourcesContent.push(originalFileContent)
+        addSource({
+          url: originalFileUrl,
+          content: originalFileContent,
+        })
       }
-
       await Promise.all(
         sources.map(async (sourceUrl, index) => {
           const contentFromSourcemap = map.sourcesContent
@@ -121,7 +142,6 @@ export const transformResultToCompilationResult = async (
     if (sourcemapExcludeSources) {
       delete map.sourcesContent
     }
-
     // we don't need sourceRoot because our path are relative or absolute to the current location
     // we could comment this line because it is not set by babel because not passed during transform
     delete map.sourceRoot
@@ -139,22 +159,28 @@ export const transformResultToCompilationResult = async (
         compiledFileUrl,
       )
       output = setSourceMappingUrl(output, sourcemapFileRelativePathForModule)
-      assets.push(sourcemapFileUrl)
-      assetsContent.push(stringifyMap(map))
+      addAsset({
+        url: sourcemapFileUrl,
+        content: stringifyMap(map),
+      })
     }
   } else {
-    sources.push(originalFileUrl)
-    sourcesContent.push(originalFileContent)
+    addSource({
+      url: originalFileUrl,
+      content: originalFileContent,
+    })
   }
 
   const { coverage } = metadata
   if (coverage) {
-    const coverageAssetFileUrl = generateCompiledFileAssetUrl(
+    const coverageAssetFileUrl = generateCompilationAssetUrl(
       compiledFileUrl,
       "coverage.json",
     )
-    assets.push(coverageAssetFileUrl)
-    assetsContent.push(stringifyCoverage(coverage))
+    addAsset({
+      url: coverageAssetFileUrl,
+      content: stringifyCoverage(coverage),
+    })
   }
 
   const { dependencies = [] } = metadata
@@ -179,18 +205,15 @@ const resolveSourceFile = ({
   projectDirectoryUrl,
 }) => {
   const sourceFileUrl = resolveSourceUrl({ source, sourcemapFileUrl })
-
-  if (!sourceFileUrl.startsWith(projectDirectoryUrl)) {
+  if (!urlIsInsideOf(sourceFileUrl, projectDirectoryUrl)) {
     // do not track dependency outside project
     // it means cache stays valid for those external sources
     return null
   }
-
   const fileFound = testFilePresence(sourceFileUrl)
   if (fileFound) {
     return sourceFileUrl
   }
-
   // prefer original source file
   const relativeUrl = urlToRelativeUrl(sourceFileUrl, compiledFileUrl)
   const originalSourceUrl = resolveUrl(relativeUrl, originalFileUrl)
@@ -211,7 +234,6 @@ const resolveSourceUrl = ({ source, sourcemapFileUrl }) => {
     )
     return ensureWindowsDriveLetter(url, sourcemapFileUrl)
   }
-
   return resolveUrl(source, sourcemapFileUrl)
 }
 

package/src/internal/executing/executePlan.js

@@ -51,6 +51,7 @@ export const executePlan = async (
     compileServerCanWriteOnFilesystem,
     babelPluginMap,
     babelConfigFileUrl,
+    preservedUrls,
     workers,
     serviceWorkers,
     importMapInWebWorkers,
@@ -126,6 +127,7 @@ export const executePlan = async (
       keepProcessAlive: true, // to be sure it stays alive
       babelPluginMap,
       babelConfigFileUrl,
+      preservedUrls,
       workers,
       serviceWorkers,
       importMapInWebWorkers,

package/src/internal/fetchUrl.js

@@ -13,9 +13,9 @@ export const fetchUrl = async (
     ignoreHttpsError,
     ...rest,
   })
-
-  return {
+  const responseObject = {
     url: response.url,
+    type: "default",
     status: response.status,
     statusText: response.statusText,
     headers: headersToObject(response.headers),
@@ -24,4 +24,5 @@ export const fetchUrl = async (
     blob: response.blob.bind(response),
     arrayBuffer: response.arrayBuffer.bind(response),
   }
+  return responseObject
 }

package/src/internal/jsenv_remote_directory.js

@@ -0,0 +1,156 @@
+import {
+  urlIsInsideOf,
+  urlToRelativeUrl,
+  urlToRessource,
+  normalizeStructuredMetaMap,
+  urlToMeta,
+  writeFile,
+} from "@jsenv/filesystem"
+import { validateResponseIntegrity } from "@jsenv/integrity"
+
+import { fetchUrl } from "@jsenv/core/src/internal/fetchUrl.js"
+
+import { originDirectoryConverter } from "./origin_directory_converter.js"
+
+export const createJsenvRemoteDirectory = ({
+  projectDirectoryUrl,
+  jsenvDirectoryRelativeUrl,
+  preservedUrls,
+}) => {
+  const jsenvRemoteDirectoryUrl = `${projectDirectoryUrl}${jsenvDirectoryRelativeUrl}.remote/`
+  const structuredMetaMap = normalizeStructuredMetaMap(
+    { preserved: preservedUrls },
+    projectDirectoryUrl,
+  )
+  const isPreservedUrl = (url) => {
+    const meta = urlToMeta({ url, structuredMetaMap })
+    return Boolean(meta.preserved)
+  }
+
+  const jsenvRemoteDirectory = {
+    isPreservedUrl,
+
+    isRemoteUrl: (url) => {
+      return url.startsWith("http://") || url.startsWith("https://")
+    },
+
+    isFileUrlForRemoteUrl: (url) => {
+      return urlIsInsideOf(url, jsenvRemoteDirectoryUrl)
+    },
+
+    fileUrlFromRemoteUrl: (remoteUrl) => {
+      const origin = originFromUrlOrUrlPattern(remoteUrl)
+      const ressource = urlToRessource(remoteUrl)
+      const [pathname, search = ""] = ressource.split("?")
+      const directoryName = originDirectoryConverter.toDirectoryName(origin)
+      const fileRelativeUrl = `${directoryName}${
+        pathname === "" ? "/" : pathname
+      }`
+      const fileUrl = `${jsenvRemoteDirectoryUrl}${fileRelativeUrl}${search}`
+      return fileUrl
+    },
+
+    remoteUrlFromFileUrl: (fileUrl) => {
+      const fileRelativeUrl = urlToRelativeUrl(fileUrl, jsenvRemoteDirectoryUrl)
+      const firstSlashIndex = fileRelativeUrl.indexOf("/")
+      const directoryName = fileRelativeUrl.slice(0, firstSlashIndex)
+      const origin = originDirectoryConverter.fromDirectoryName(directoryName)
+      const ressource = fileRelativeUrl.slice(firstSlashIndex)
+      const remoteUrlObject = new URL(ressource, origin)
+      remoteUrlObject.searchParams.delete("integrity")
+      const remoteUrl = remoteUrlObject.href
+      return remoteUrl
+    },
+
+    loadFileUrlFromRemote: async (url, request) => {
+      const urlObject = new URL(url)
+      const integrity = urlObject.searchParams.get("integrity")
+      if (integrity) {
+        urlObject.searchParams.delete("integrity")
+      }
+      const remoteUrl = jsenvRemoteDirectory.remoteUrlFromFileUrl(
+        urlObject.href,
+      )
+      const requestHeadersToForward = { ...request.headers }
+      delete requestHeadersToForward.host
+      const response = await fetchUrl(remoteUrl, {
+        mode: "cors",
+        headers: requestHeadersToForward,
+      })
+      if (response.status !== 200) {
+        throw createRemoteFetchError({
+          code: "UNEXPECTED_STATUS",
+          message: `unexpected status for ressource "${remoteUrl}", received ${response.status}`,
+        })
+      }
+      const responseBodyAsBuffer = Buffer.from(await response.arrayBuffer())
+      if (integrity) {
+        try {
+          validateResponseIntegrity(
+            {
+              url: response.url,
+              type: response.type,
+              dataRepresentation: responseBodyAsBuffer,
+            },
+            integrity,
+          )
+        } catch (e) {
+          throw createRemoteFetchError({
+            code: e.code,
+            message: e.message,
+          })
+        }
+      }
+      await writeFile(url, responseBodyAsBuffer)
+      return responseBodyAsBuffer
+    },
+  }
+
+  return jsenvRemoteDirectory
+}
+
+const createRemoteFetchError = ({ message, code }) => {
+  const error = new Error(message)
+  error.code = "UNEXPECTED_REMOTE_URL_RESPONSE"
+  error.asResponse = () => {
+    const data = {
+      code,
+      message,
+    }
+    const json = JSON.stringify(data)
+    return {
+      status: 502,
+      statusText: "Bad Gateway",
+      headers: {
+        "cache-control": "no-store",
+        "content-length": Buffer.byteLength(json),
+        "content-type": "application/json",
+      },
+      body: json,
+    }
+  }
+  return error
+}
+
+const originFromUrlOrUrlPattern = (url) => {
+  if (url.startsWith("http://")) {
+    const slashAfterProtocol = url.indexOf("/", "http://".length + 1)
+    if (slashAfterProtocol === -1) {
+      return url
+    }
+    const origin = url.slice(0, slashAfterProtocol)
+    return origin
+  }
+  if (url.startsWith("https://")) {
+    const slashAfterProtocol = url.indexOf("/", "https://".length + 1)
+    if (slashAfterProtocol === -1) {
+      return url
+    }
+    const origin = url.slice(0, slashAfterProtocol)
+    return origin
+  }
+  if (url.startsWith("file://")) {
+    return "file://"
+  }
+  return new URL(url).origin
+}

package/src/internal/origin_directory_converter.js

@@ -0,0 +1,62 @@
+// https://stackoverflow.com/a/1184263
+// using the list of bad characters from:
+// https://github.com/parshap/node-sanitize-filename/blob/master/index.js
+// so that the directory name can be decoded
+// so that we can find back the http origin
+// we should also ignore the query string params part
+// it does not matter, it can be removed from the url and added back
+// when performing the http request
+
+const escapeChar = "$"
+const reservedChars = ["/", "?", "<", ">", "\\", ":", "*", "|", '"']
+
+export const originDirectoryConverter = {
+  toDirectoryName: (origin) => {
+    let directoryName = ""
+    let i = 0
+    while (i < origin.length) {
+      const char = origin[i]
+      i++
+      if (reservedChars.includes(char)) {
+        const charAsHex = charToHexString(char)
+        directoryName += `${escapeChar}${charAsHex}`
+      } else {
+        directoryName += char
+      }
+    }
+    return directoryName
+  },
+
+  fromDirectoryName: (filename) => {
+    let origin = ""
+    let i = 0
+    while (i < filename.length) {
+      const char = filename[i]
+      i++
+      if (char === escapeChar) {
+        const encodedChar = `${filename[i]}${filename[i + 1]}`
+        i += 2
+        const decodedChar = charFromHexString(encodedChar)
+        origin += decodedChar
+      } else {
+        origin += char
+      }
+    }
+    return origin
+  },
+}
+
+const charToHexString = (char) => {
+  const charCode = char.charCodeAt(0)
+  const hexString = charCode.toString(16)
+  if (charCode < 16) {
+    return `0${hexString}`
+  }
+  return hexString
+}
+
+const charFromHexString = (hexString) => {
+  const charCode = parseInt(hexString, 16)
+  const char = String.fromCharCode(charCode)
+  return char
+}