@jsenv/core 25.1.1 → 25.2.0

package/src/internal/compiling/jsenvCompilerForJavaScript.js

@@ -1,12 +1,13 @@
+import { generateSourcemapUrl } from "@jsenv/core/src/internal/sourceMappingURLUtils.js"
+
 import { transformJs } from "./js-compilation-service/transformJs.js"
 import { transformResultToCompilationResult } from "./transformResultToCompilationResult.js"
 
 export const compileJavascript = async ({
-  code,
-  map,
+  projectDirectoryUrl,
+  jsenvRemoteDirectory,
   url,
   compiledUrl,
-  projectDirectoryUrl,
 
   babelPluginMap,
   workerUrls,
@@ -16,6 +17,8 @@ export const compileJavascript = async ({
   topLevelAwait,
   prependSystemJs,
 
+  code,
+  map,
   sourcemapExcludeSources,
   sourcemapMethod,
 }) => {
@@ -23,38 +26,39 @@ export const compileJavascript = async ({
     prependSystemJs =
       workerUrls.includes(url) || serviceWorkerUrls.includes(url)
   }
-
   const transformResult = await transformJs({
-    code,
-    map,
+    projectDirectoryUrl,
+    jsenvRemoteDirectory,
     url,
     compiledUrl,
-    projectDirectoryUrl,
 
     babelPluginMap,
     moduleOutFormat,
     importMetaFormat,
     topLevelAwait,
     prependSystemJs,
-  })
 
+    code,
+    map,
+  })
   return transformResultToCompilationResult(
     {
       contentType: "application/javascript",
+      metadata: transformResult.metadata,
       code: transformResult.code,
       map: transformResult.map,
-      metadata: transformResult.metadata,
     },
     {
       projectDirectoryUrl,
-      originalFileContent: code,
+      jsenvRemoteDirectory,
       originalFileUrl: url,
       compiledFileUrl: compiledUrl,
       // sourcemap are not inside the asset folder because
       // of https://github.com/microsoft/vscode-chrome-debug-core/issues/544
-      sourcemapFileUrl: `${compiledUrl}.map`,
+      sourcemapFileUrl: generateSourcemapUrl(compiledUrl),
       sourcemapExcludeSources,
       sourcemapMethod,
+      originalFileContent: code,
     },
   )
 }

package/src/internal/compiling/startCompileServer.js

@@ -36,17 +36,19 @@ import {
 } from "@jsenv/core/dist/build_manifest.js"
 import { generateGroupMap } from "@jsenv/core/src/internal/generateGroupMap/generateGroupMap.js"
 import { isBrowserPartOfSupportedRuntimes } from "@jsenv/core/src/internal/generateGroupMap/runtime_support.js"
-import { loadBabelPluginMapFromFile } from "./load_babel_plugin_map_from_file.js"
-import { extractSyntaxBabelPluginMap } from "./babel_plugins.js"
+
+import { createJsenvRemoteDirectory } from "../jsenv_remote_directory.js"
 import {
   sourcemapMainFileInfo,
   sourcemapMappingFileInfo,
 } from "../jsenvInternalFiles.js"
+import { babelPluginReplaceExpressions } from "../babel_plugin_replace_expressions.js"
 import {
   jsenvCoreDirectoryUrl,
   jsenvDistDirectoryUrl,
 } from "../jsenvCoreDirectoryUrl.js"
-import { babelPluginReplaceExpressions } from "../babel_plugin_replace_expressions.js"
+import { loadBabelPluginMapFromFile } from "./load_babel_plugin_map_from_file.js"
+import { extractSyntaxBabelPluginMap } from "./babel_plugins.js"
 import { babelPluginGlobalThisAsJsenvImport } from "./babel_plugin_global_this_as_jsenv_import.js"
 import { babelPluginNewStylesheetAsJsenvImport } from "./babel_plugin_new_stylesheet_as_jsenv_import.js"
 import { babelPluginImportAssertions } from "./babel_plugin_import_assertions.js"
@@ -98,6 +100,7 @@ export const startCompileServer = async ({
   babelPluginMap,
   babelConfigFileUrl,
   customCompilers = {},
+  preservedUrls,
   workers = [],
   serviceWorkers = [],
   importMapInWebWorkers = false,
@@ -146,6 +149,30 @@ export const startCompileServer = async ({
   )
   const logger = createLogger({ logLevel })
 
+  preservedUrls = {
+    // Authorize jsenv to modify any file url
+    // because the goal is to build the files into chunks
+    "file://": false,
+    // Preserves http and https urls
+    // because if code specifiy a CDN url it's usually because code wants
+    // to keep the url intact and keep HTTP request to CDN (both in dev and prod)
+    "http://": true,
+    "https://": true,
+    /*
+     * It's possible to selectively overrides the behaviour above:
+     * 1. The CDN file needs to be transformed to be executable in dev, build or both
+     * preservedUrls: {"https://cdn.skypack.dev/preact@10.6.4": false}
+     * 2. No strong need to preserve the CDN dependency
+     * 3. Prevent concatenation of a file during build
+     * preservedUrls: {"./file.js": false}
+     */
+    ...preservedUrls,
+  }
+  const jsenvRemoteDirectory = createJsenvRemoteDirectory({
+    projectDirectoryUrl,
+    jsenvDirectoryRelativeUrl,
+    preservedUrls,
+  })
   const workerUrls = workers.map((worker) =>
     resolveUrl(worker, projectDirectoryUrl),
   )
@@ -290,11 +317,18 @@ export const startCompileServer = async ({
     projectDirectoryUrl,
     jsenvDirectoryRelativeUrl,
     outDirectoryRelativeUrl,
+    jsenvRemoteDirectory,
     importDefaultExtension,
+
+    preservedUrls,
+    workers,
+    serviceWorkers,
     compileServerGroupMap,
+    babelPluginMap,
+    replaceProcessEnvNodeEnv,
+    processEnvNodeEnv,
     env,
     inlineImportMapIntoHTML,
-    babelPluginMap,
     customCompilers,
     jsenvToolbarInjection,
     sourcemapMethod,
@@ -328,7 +362,9 @@ export const startCompileServer = async ({
       logger,
 
       projectDirectoryUrl,
+      jsenvDirectoryRelativeUrl,
       outDirectoryRelativeUrl,
+      jsenvRemoteDirectory,
 
       importDefaultExtension,
 
@@ -369,6 +405,7 @@ export const startCompileServer = async ({
       : {}),
     "service:source file": createSourceFileService({
       projectDirectoryUrl,
+      jsenvRemoteDirectory,
       projectFileRequestedCallback,
       projectFileCacheStrategy,
     }),
@@ -424,6 +461,7 @@ export const startCompileServer = async ({
     ...compileServer,
     compileServerGroupMap,
     babelPluginMap,
+    preservedUrls,
     projectFileRequestedCallback,
   }
 }
@@ -939,30 +977,45 @@ const createSourceFileService = ({
   projectDirectoryUrl,
   projectFileRequestedCallback,
   projectFileCacheStrategy,
+  jsenvRemoteDirectory,
 }) => {
   return async (request) => {
     const relativeUrl = request.pathname.slice(1)
     projectFileRequestedCallback(relativeUrl, request)
-
     const fileUrl = new URL(request.ressource.slice(1), projectDirectoryUrl)
       .href
     const fileIsInsideJsenvDistDirectory = urlIsInsideOf(
       fileUrl,
       jsenvDistDirectoryUrl,
     )
-
-    const responsePromise = fetchFileSystem(fileUrl, {
-      headers: request.headers,
-      etagEnabled: projectFileCacheStrategy === "etag",
-      mtimeEnabled: projectFileCacheStrategy === "mtime",
-      ...(fileIsInsideJsenvDistDirectory
-        ? {
-            cacheControl: `private,max-age=${60 * 60 * 24 * 30},immutable`,
-          }
-        : {}),
-    })
-
-    return responsePromise
+    const fromFileSystem = () =>
+      fetchFileSystem(fileUrl, {
+        headers: request.headers,
+        etagEnabled: projectFileCacheStrategy === "etag",
+        mtimeEnabled: projectFileCacheStrategy === "mtime",
+        ...(fileIsInsideJsenvDistDirectory
+          ? {
+              cacheControl: `private,max-age=${60 * 60 * 24 * 30},immutable`,
+            }
+          : {}),
+      })
+    const filesystemResponse = await fromFileSystem()
+    if (
+      filesystemResponse.status === 404 &&
+      jsenvRemoteDirectory.isFileUrlForRemoteUrl(fileUrl)
+    ) {
+      try {
+        await jsenvRemoteDirectory.loadFileUrlFromRemote(fileUrl, request)
+        // re-fetch filesystem instead to ensure response headers are correct
+        return fromFileSystem()
+      } catch (e) {
+        if (e && e.asResponse) {
+          return e.asResponse()
+        }
+        throw e
+      }
+    }
+    return filesystemResponse
   }
 }
 
@@ -971,6 +1024,10 @@ const createCompileServerMetaFileInfo = ({
   jsenvDirectoryRelativeUrl,
   outDirectoryRelativeUrl,
   importDefaultExtension,
+
+  preservedUrls,
+  workers,
+  serviceWorkers,
   compileServerGroupMap,
   babelPluginMap,
   replaceProcessEnvNodeEnv,
@@ -1016,6 +1073,9 @@ const createCompileServerMetaFileInfo = ({
     outDirectoryRelativeUrl,
     importDefaultExtension,
 
+    preservedUrls,
+    workers,
+    serviceWorkers,
     babelPluginMap: babelPluginMapAsData(babelPluginMap),
     compileServerGroupMap,
     customCompilerPatterns,
@@ -1029,7 +1089,7 @@ const createCompileServerMetaFileInfo = ({
     sourcemapMappingFileRelativeUrl,
     errorStackRemapping: true,
 
-    // used to consider the logic generating files may have changed
+    // used to consider logic generating files may have changed
     jsenvCorePackageVersion,
 
     // impact only HTML files

package/src/internal/compiling/transformResultToCompilationResult.js

@@ -3,6 +3,7 @@ import {
   urlToRelativeUrl,
   readFile,
   ensureWindowsDriveLetter,
+  urlIsInsideOf,
 } from "@jsenv/filesystem"
 
 import {
@@ -15,16 +16,16 @@ import {
   setCssSourceMappingUrl,
   sourcemapToBase64Url,
 } from "../sourceMappingURLUtils.js"
-import { generateCompiledFileAssetUrl } from "./compile-directory/compile-asset.js"
+import { generateCompilationAssetUrl } from "./compile-directory/compile-asset.js"
 import { testFilePresence } from "./compile-directory/fs-optimized-for-cache.js"
 
 const isWindows = process.platform === "win32"
 
 export const transformResultToCompilationResult = async (
-  { contentType, code, map, metadata = {} },
+  { contentType, metadata = {}, code, map },
   {
     projectDirectoryUrl,
-    originalFileContent,
+    jsenvRemoteDirectory,
     originalFileUrl,
     compiledFileUrl,
     sourcemapFileUrl,
@@ -33,9 +34,13 @@ export const transformResultToCompilationResult = async (
     // it also means client have to fetch source from server (additional http request)
     // some client ignore sourcesContent property such as vscode-chrome-debugger
     // Because it's the most complex scenario and we want to ensure client is always able
-    // to find source from the sourcemap, we remove map.sourcesContent by default to test this.
-    sourcemapExcludeSources = true,
+    // to find source from the sourcemap, it's a good idea
+    // to exclude sourcesContent from sourcemap.
+    // However some ressource are abstract and it means additional http request for the browser.
+    // For these reasons it's simpler to keep source content in sourcemap.
+    sourcemapExcludeSources = false,
     sourcemapMethod = "comment", // "comment", "inline"
+    originalFileContent,
   },
 ) => {
   if (typeof contentType !== "string") {
@@ -71,6 +76,14 @@ export const transformResultToCompilationResult = async (
   const sourcesContent = []
   const assets = []
   const assetsContent = []
+  const addSource = ({ url, content }) => {
+    sources.push(url)
+    sourcesContent.push(content)
+  }
+  const addAsset = ({ url, content }) => {
+    assets.push(url)
+    assetsContent.push(content)
+  }
 
   let output = code
   if (sourcemapEnabled && map) {
@@ -78,8 +91,10 @@ export const transformResultToCompilationResult = async (
       // may happen in some cases where babel returns a wrong sourcemap
       // there is at least one case where it happens
       // a file with only import './whatever.js' inside
-      sources.push(originalFileUrl)
-      sourcesContent.push(originalFileContent)
+      addSource({
+        url: originalFileUrl,
+        content: originalFileContent,
+      })
     } else {
       map.sources.forEach((source, index) => {
         const sourceFileUrl = resolveSourceFile({
@@ -90,19 +105,25 @@ export const transformResultToCompilationResult = async (
           projectDirectoryUrl,
         })
         if (sourceFileUrl) {
-          map.sources[index] = urlToRelativeUrl(sourceFileUrl, sourcemapFileUrl)
+          // In case the file comes from a remote url
+          // we prefer to consider remote url as the real source for this code
+          map.sources[index] =
+            jsenvRemoteDirectory &&
+            jsenvRemoteDirectory.isFileUrlForRemoteUrl(sourceFileUrl)
+              ? jsenvRemoteDirectory.remoteUrlFromFileUrl(sourceFileUrl)
+              : urlToRelativeUrl(sourceFileUrl, sourcemapFileUrl)
           sources[index] = sourceFileUrl
         }
       })
-
       if (sources.length === 0) {
         // happens when sourcemap is generated by webpack and looks like
         // webpack://Package./src/file.js
         // in that case we'll don't know how to find the source file
-        sources.push(originalFileUrl)
-        sourcesContent.push(originalFileContent)
+        addSource({
+          url: originalFileUrl,
+          content: originalFileContent,
+        })
       }
-
       await Promise.all(
         sources.map(async (sourceUrl, index) => {
           const contentFromSourcemap = map.sourcesContent
@@ -121,7 +142,6 @@ export const transformResultToCompilationResult = async (
     if (sourcemapExcludeSources) {
       delete map.sourcesContent
     }
-
     // we don't need sourceRoot because our path are relative or absolute to the current location
     // we could comment this line because it is not set by babel because not passed during transform
     delete map.sourceRoot
@@ -139,22 +159,28 @@ export const transformResultToCompilationResult = async (
         compiledFileUrl,
       )
       output = setSourceMappingUrl(output, sourcemapFileRelativePathForModule)
-      assets.push(sourcemapFileUrl)
-      assetsContent.push(stringifyMap(map))
+      addAsset({
+        url: sourcemapFileUrl,
+        content: stringifyMap(map),
+      })
     }
   } else {
-    sources.push(originalFileUrl)
-    sourcesContent.push(originalFileContent)
+    addSource({
+      url: originalFileUrl,
+      content: originalFileContent,
+    })
   }
 
   const { coverage } = metadata
   if (coverage) {
-    const coverageAssetFileUrl = generateCompiledFileAssetUrl(
+    const coverageAssetFileUrl = generateCompilationAssetUrl(
       compiledFileUrl,
       "coverage.json",
     )
-    assets.push(coverageAssetFileUrl)
-    assetsContent.push(stringifyCoverage(coverage))
+    addAsset({
+      url: coverageAssetFileUrl,
+      content: stringifyCoverage(coverage),
+    })
   }
 
   const { dependencies = [] } = metadata
@@ -179,18 +205,15 @@ const resolveSourceFile = ({
   projectDirectoryUrl,
 }) => {
   const sourceFileUrl = resolveSourceUrl({ source, sourcemapFileUrl })
-
-  if (!sourceFileUrl.startsWith(projectDirectoryUrl)) {
+  if (!urlIsInsideOf(sourceFileUrl, projectDirectoryUrl)) {
     // do not track dependency outside project
     // it means cache stays valid for those external sources
     return null
   }
-
   const fileFound = testFilePresence(sourceFileUrl)
   if (fileFound) {
     return sourceFileUrl
   }
-
   // prefer original source file
   const relativeUrl = urlToRelativeUrl(sourceFileUrl, compiledFileUrl)
   const originalSourceUrl = resolveUrl(relativeUrl, originalFileUrl)
@@ -211,7 +234,6 @@ const resolveSourceUrl = ({ source, sourcemapFileUrl }) => {
     )
     return ensureWindowsDriveLetter(url, sourcemapFileUrl)
   }
-
   return resolveUrl(source, sourcemapFileUrl)
 }
 

package/src/internal/executing/executePlan.js

@@ -51,6 +51,7 @@ export const executePlan = async (
     compileServerCanWriteOnFilesystem,
     babelPluginMap,
     babelConfigFileUrl,
+    preservedUrls,
     workers,
     serviceWorkers,
     importMapInWebWorkers,
@@ -126,6 +127,7 @@ export const executePlan = async (
       keepProcessAlive: true, // to be sure it stays alive
       babelPluginMap,
       babelConfigFileUrl,
+      preservedUrls,
       workers,
       serviceWorkers,
       importMapInWebWorkers,

package/src/internal/fetchUrl.js

@@ -13,9 +13,9 @@ export const fetchUrl = async (
     ignoreHttpsError,
     ...rest,
   })
-
-  return {
+  const responseObject = {
     url: response.url,
+    type: "default",
     status: response.status,
     statusText: response.statusText,
     headers: headersToObject(response.headers),
@@ -24,4 +24,5 @@ export const fetchUrl = async (
     blob: response.blob.bind(response),
     arrayBuffer: response.arrayBuffer.bind(response),
   }
+  return responseObject
 }

package/src/internal/integrity/integrity_algorithms.js

@@ -0,0 +1,26 @@
+import crypto from "node:crypto"
+
+export const isSupportedAlgorithm = (algo) => {
+  return SUPPORTED_ALGORITHMS.includes(algo)
+}
+
+// https://www.w3.org/TR/SRI/#priority
+export const getPrioritizedHashFunction = (firstAlgo, secondAlgo) => {
+  const firstIndex = SUPPORTED_ALGORITHMS.indexOf(firstAlgo)
+  const secondIndex = SUPPORTED_ALGORITHMS.indexOf(secondAlgo)
+  if (firstIndex === secondIndex) {
+    return ""
+  }
+  if (firstIndex < secondIndex) {
+    return secondAlgo
+  }
+  return firstAlgo
+}
+
+export const applyAlgoToRepresentationData = (algo, data) => {
+  const base64Value = crypto.createHash(algo).update(data).digest("base64")
+  return base64Value
+}
+
+// keep this ordered by collision resistance as it is also used by "getPrioritizedHashFunction"
+const SUPPORTED_ALGORITHMS = ["sha256", "sha384", "sha512"]

package/src/internal/integrity/integrity_parsing.js

@@ -0,0 +1,50 @@
+import { isSupportedAlgorithm } from "./integrity_algorithms.js"
+
+// see https://w3c.github.io/webappsec-subresource-integrity/#parse-metadata
+export const parseIntegrity = (string) => {
+  const integrityMetadata = {}
+  string
+    .trim()
+    .split(/\s+/)
+    .forEach((token) => {
+      const { isValid, algo, base64Value, optionExpression } =
+        parseAsHashWithOptions(token)
+      if (!isValid) {
+        return
+      }
+      if (!isSupportedAlgorithm(algo)) {
+        return
+      }
+      const metadataList = integrityMetadata[algo]
+      const metadata = { base64Value, optionExpression }
+      integrityMetadata[algo] = metadataList
+        ? [...metadataList, metadata]
+        : [metadata]
+    })
+  return integrityMetadata
+}
+
+// see https://w3c.github.io/webappsec-subresource-integrity/#the-integrity-attribute
+const parseAsHashWithOptions = (token) => {
+  const dashIndex = token.indexOf("-")
+  if (dashIndex === -1) {
+    return { isValid: false }
+  }
+  const beforeDash = token.slice(0, dashIndex)
+  const afterDash = token.slice(dashIndex + 1)
+  const questionIndex = afterDash.indexOf("?")
+  const algo = beforeDash
+  if (questionIndex === -1) {
+    const base64Value = afterDash
+    const isValid = BASE64_REGEX.test(afterDash)
+    return { isValid, algo, base64Value }
+  }
+  const base64Value = afterDash.slice(0, questionIndex)
+  const optionExpression = afterDash.slice(questionIndex + 1)
+  const isValid =
+    BASE64_REGEX.test(afterDash) && VCHAR_REGEX.test(optionExpression)
+  return { isValid, algo, base64Value, optionExpression }
+}
+
+const BASE64_REGEX = /^[A-Za-z0-9+\/=+]+$/
+const VCHAR_REGEX = /^[\x21-\x7E]+$/

package/src/internal/integrity/integrity_update.js

@@ -0,0 +1,23 @@
+import { parseIntegrity } from "./integrity_parsing.js"
+import {
+  getPrioritizedHashFunction,
+  applyAlgoToRepresentationData,
+} from "./integrity_algorithms.js"
+
+export const updateIntegrity = (integrity, representationData) => {
+  const integrityMetadata = parseIntegrity(integrity)
+  const algos = Object.keys(integrityMetadata)
+  if (algos.length === 0) {
+    return ""
+  }
+  let strongestAlgo = algos[0]
+  algos.slice(1).forEach((algoCandidate) => {
+    strongestAlgo =
+      getPrioritizedHashFunction(strongestAlgo, algoCandidate) || strongestAlgo
+  })
+  const base64Value = applyAlgoToRepresentationData(
+    strongestAlgo,
+    representationData,
+  )
+  return `${strongestAlgo}-${base64Value}`
+}

package/src/internal/integrity/integrity_validation.js

@@ -0,0 +1,49 @@
+import { parseIntegrity } from "./integrity_parsing.js"
+import {
+  getPrioritizedHashFunction,
+  applyAlgoToRepresentationData,
+} from "./integrity_algorithms.js"
+
+// https://www.w3.org/TR/SRI/#does-response-match-metadatalist
+export const validateResponseIntegrity = (
+  { url, type, dataRepresentation },
+  integrity,
+) => {
+  if (!isResponseEligibleForIntegrityValidation({ type })) {
+    return false
+  }
+  const integrityMetadata = parseIntegrity(integrity)
+  const algos = Object.keys(integrityMetadata)
+  if (algos.length === 0) {
+    return true
+  }
+  let strongestAlgo = algos[0]
+  algos.slice(1).forEach((algoCandidate) => {
+    strongestAlgo =
+      getPrioritizedHashFunction(strongestAlgo, algoCandidate) || strongestAlgo
+  })
+  const metadataList = integrityMetadata[strongestAlgo]
+  const actualBase64Value = applyAlgoToRepresentationData(
+    strongestAlgo,
+    dataRepresentation,
+  )
+  const acceptedBase64Values = metadataList.map(
+    (metadata) => metadata.base64Value,
+  )
+  const someIsMatching = acceptedBase64Values.includes(actualBase64Value)
+  if (someIsMatching) {
+    return true
+  }
+  const error = new Error(
+    `Integrity validation failed for ressource "${url}". The integrity found for this ressource is "${strongestAlgo}-${actualBase64Value}"`,
+  )
+  error.code = "EINTEGRITY"
+  error.algorithm = strongestAlgo
+  error.found = actualBase64Value
+  throw error
+}
+
+// https://www.w3.org/TR/SRI/#is-response-eligible-for-integrity-validation
+const isResponseEligibleForIntegrityValidation = (response) => {
+  return ["basic", "cors", "default"].includes(response.type)
+}