@jsenv/core 25.1.1 → 25.3.0

package/src/internal/building/url_fetcher.js

@@ -5,6 +5,7 @@ import { fetchUrl as jsenvFetchUrl } from "@jsenv/core/src/internal/fetchUrl.js"
 import { validateResponse } from "@jsenv/core/src/internal/response_validation.js"
 
 export const createUrlFetcher = ({
+  jsenvRemoteDirectory,
   asOriginalUrl,
   asProjectUrl,
   applyUrlMappings,
@@ -13,18 +14,28 @@ export const createUrlFetcher = ({
 }) => {
   const urlRedirectionMap = {}
 
-  const fetchUrl = async (url, { signal, urlTrace, contentTypeExpected }) => {
+  const fetchUrl = async (
+    url,
+    { signal, urlTrace, contentTypeExpected, crossorigin },
+  ) => {
     const urlToFetch = applyUrlMappings(url)
-
     const response = await jsenvFetchUrl(urlToFetch, {
       signal,
       ignoreHttpsError: true,
+      credentials:
+        crossorigin === "anonymous" || crossorigin === ""
+          ? "same-origin"
+          : crossorigin === "use-credentials"
+          ? "include"
+          : undefined,
     })
     const responseUrl = response.url
-
+    const originalUrl =
+      asOriginalUrl(responseUrl) || asProjectUrl(responseUrl) || responseUrl
     const responseValidity = await validateResponse(response, {
-      originalUrl:
-        asOriginalUrl(responseUrl) || asProjectUrl(responseUrl) || responseUrl,
+      originalUrl: jsenvRemoteDirectory.isFileUrlForRemoteUrl(originalUrl)
+        ? jsenvRemoteDirectory.remoteUrlFromFileUrl(originalUrl)
+        : originalUrl,
       urlTrace,
       contentTypeExpected,
     })
@@ -56,11 +67,9 @@ export const createUrlFetcher = ({
       beforeThrowingResponseValidationError(responseValidationError)
       throw responseValidationError
     }
-
     if (url !== responseUrl) {
       urlRedirectionMap[url] = responseUrl
     }
-
     return response
   }
 

package/src/internal/building/url_loader.js

@@ -8,7 +8,6 @@ export const createUrlLoader = ({
   allowJson,
   urlImporterMap,
 
-  asServerUrl,
   asProjectUrl,
   asOriginalUrl,
 
@@ -16,15 +15,12 @@ export const createUrlLoader = ({
 }) => {
   const urlResponseBodyMap = {}
 
-  const loadUrl = async (rollupUrl, { signal, logger }) => {
-    let url = asServerUrl(rollupUrl)
-
+  const loadUrl = async (url, { signal, logger }) => {
     const customLoader = urlCustomLoaders[url]
     if (customLoader) {
       const result = await customLoader()
       return result
     }
-
     const response = await urlFetcher.fetchUrl(url, {
       signal,
       contentTypeExpected: [

package/src/internal/building/url_versioning.js

@@ -1,177 +1,4 @@
 import { createHash } from "crypto"
-import {
-  urlToParentUrl,
-  urlToBasename,
-  urlToExtension,
-} from "@jsenv/filesystem"
-
-export const createUrlVersioner = ({
-  entryPointUrls,
-  asOriginalUrl,
-  lineBreakNormalization,
-}) => {
-  const availableNameGenerator = createAvailableNameGenerator()
-
-  const computeBuildRelativeUrl = (ressource, precomputation) => {
-    const pattern = getFilenamePattern({
-      ressource,
-      entryPointUrls,
-      asOriginalUrl,
-      precomputeBuildRelativeUrl,
-    })
-
-    const buildRelativeUrl = generateBuildRelativeUrl(
-      ressource.url,
-      ressource.bufferAfterBuild,
-      {
-        pattern,
-        getName: precomputation
-          ? () => urlToBasename(ressource.url)
-          : () =>
-              availableNameGenerator.getAvailableNameForPattern(
-                urlToBasename(ressource.url),
-                pattern,
-              ),
-        contentType: ressource.contentType,
-        lineBreakNormalization,
-      },
-    )
-    return buildRelativeUrl
-  }
-
-  const precomputeBuildRelativeUrl = (ressource, bufferAfterBuild = "") => {
-    if (ressource.buildRelativeUrl) {
-      return ressource.buildRelativeUrl
-    }
-    if (ressource.precomputedBuildRelativeUrl) {
-      return ressource.precomputedBuildRelativeUrl
-    }
-
-    ressource.bufferAfterBuild = bufferAfterBuild
-    const precomputedBuildRelativeUrl = computeBuildRelativeUrl(ressource, true)
-    ressource.bufferAfterBuild = undefined
-    ressource.precomputedBuildRelativeUrl = precomputedBuildRelativeUrl
-    return precomputedBuildRelativeUrl
-  }
-
-  return {
-    computeBuildRelativeUrl,
-    precomputeBuildRelativeUrl,
-  }
-}
-
-const createAvailableNameGenerator = () => {
-  const cache = {}
-  const getAvailableNameForPattern = (name, pattern) => {
-    let names = cache[pattern]
-    if (!names) {
-      names = []
-      cache[pattern] = names
-    }
-
-    let nameCandidate = name
-    let integer = 1
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      if (!names.includes(nameCandidate)) {
-        names.push(nameCandidate)
-        return nameCandidate
-      }
-      integer++
-      nameCandidate = `${name}${integer}`
-    }
-  }
-
-  return {
-    getAvailableNameForPattern,
-  }
-}
-
-const getFilenamePattern = ({
-  ressource,
-  entryPointUrls,
-  asOriginalUrl,
-  precomputeBuildRelativeUrl,
-}) => {
-  if (ressource.isEntryPoint) {
-    const originalUrl = asOriginalUrl(ressource.url)
-    const entryPointBuildRelativeUrl = entryPointUrls[originalUrl]
-    return entryPointBuildRelativeUrl
-  }
-
-  // inline ressource inherits location
-  if (ressource.isInline) {
-    // inherit parent directory location because it's an inline file
-    const importerBuildRelativeUrl = precomputeBuildRelativeUrl(
-      ressource.importer,
-    )
-    const name = urlToBasename(`file://${importerBuildRelativeUrl}`)
-    return `${name}_[hash][extname]`
-  }
-
-  // importmap.
-  // we want to force the fileName for the importmap
-  // so that we don't have to rewrite its content
-  // the goal is to put the importmap at the same relative path
-  // than in the project
-  if (ressource.contentType === "application/importmap+json") {
-    const importmapRessourceUrl = ressource.url
-    const name = urlToBasename(importmapRessourceUrl)
-    return `${name}_[hash][extname]`
-  }
-
-  // service worker:
-  // - MUST be at the root (same level than the HTML file)
-  //   otherwise it might be registered for the scope "/assets/" instead of "/"
-  // - MUST not be versioned
-  if (ressource.isServiceWorker) {
-    return "[name][extname]"
-  }
-
-  if (ressource.isWorker) {
-    return "[name]_[hash][extname]"
-  }
-
-  if (ressource.isJsModule) {
-    return "[name]_[hash].js"
-  }
-
-  return ressource.urlVersioningDisabled
-    ? "assets/[name][extname]"
-    : "assets/[name]_[hash][extname]"
-}
-
-const generateBuildRelativeUrl = (
-  fileUrl,
-  fileContent,
-  {
-    getName,
-    pattern,
-    contentType = "application/octet-stream",
-    lineBreakNormalization = false,
-  } = {},
-) => {
-  pattern = typeof pattern === "function" ? pattern() : pattern
-  if (pattern.startsWith("./")) pattern = pattern.slice(2)
-  const buildRelativeUrl = renderFileNamePattern(pattern, {
-    dirname: () => urlToParentUrl(fileUrl),
-    name: getName,
-    hash: () =>
-      generateContentHash(fileContent, {
-        contentType,
-        lineBreakNormalization,
-      }),
-    extname: () => urlToExtension(fileUrl),
-  })
-  return buildRelativeUrl
-}
-
-const renderFileNamePattern = (pattern, replacements) => {
-  return pattern.replace(/\[(\w+)\]/g, (_match, type) => {
-    const replacement = replacements[type]()
-    return replacement
-  })
-}
 
 // https://github.com/rollup/rollup/blob/19e50af3099c2f627451a45a84e2fa90d20246d5/src/utils/FileEmitter.ts#L47
 export const generateContentHash = (

package/src/internal/compiling/babel_plugin_import_metadata.js

@@ -2,21 +2,17 @@ import { babelPluginImportVisitor } from "./babel_plugin_import_visitor.js"
 
 export const babelPluginImportMetadata = (babel) => {
   return {
-    ...babelPluginImportVisitor(
-      babel,
-      // During the build we throw when for import call expression where
-      // sepcifier or type is dynamic.
-      // Here there is no strong need to throw because keeping the source code intact
-      // will throw an error when browser will execute the code
-      ({ state, specifierPath }) => {
+    ...babelPluginImportVisitor(babel, ({ state, specifierPath }) => {
+      const specifierNode = specifierPath.node
+      if (specifierNode.type === "StringLiteral") {
+        const specifier = specifierNode.value
         const { metadata } = state.file
-
         metadata.dependencies = [
           ...(metadata.dependencies ? metadata.dependencies : []),
-          specifierPath.node.value,
+          specifier,
         ]
-      },
-    ),
+      }
+    }),
     name: "import-metadata",
   }
 }

package/src/internal/compiling/babel_plugin_proxy_external_imports.js

@@ -0,0 +1,31 @@
+import { urlToRelativeUrl, fileSystemPathToUrl } from "@jsenv/filesystem"
+
+import { babelPluginImportVisitor } from "./babel_plugin_import_visitor.js"
+
+export const babelPluginProxyExternalImports = (
+  babel,
+  { jsenvRemoteDirectory },
+) => {
+  return {
+    ...babelPluginImportVisitor(babel, ({ specifierPath, state }) => {
+      const specifierNode = specifierPath.node
+      if (specifierNode.type === "StringLiteral") {
+        const specifier = specifierNode.value
+        if (
+          jsenvRemoteDirectory.isRemoteUrl(specifier) &&
+          !jsenvRemoteDirectory.isPreservedUrl(specifier)
+        ) {
+          const fileUrl = jsenvRemoteDirectory.fileUrlFromRemoteUrl(specifier)
+          const importerFileUrl = fileSystemPathToUrl(state.filename)
+          const urlRelativeToProject = urlToRelativeUrl(
+            fileUrl,
+            importerFileUrl,
+          )
+          const specifierProxy = `./${urlRelativeToProject}`
+          specifierPath.replaceWith(babel.types.stringLiteral(specifierProxy))
+        }
+      }
+    }),
+    name: "proxy-external-imports",
+  }
+}

package/src/internal/compiling/compile-directory/compile-asset.js

@@ -1,4 +1,4 @@
-import { urlToFilename } from "@jsenv/filesystem"
+import { urlToFilename, urlToOrigin, urlToPathname } from "@jsenv/filesystem"
 
 export const urlIsCompilationAsset = (url) => {
   const filename = urlToFilename(url)
@@ -13,8 +13,12 @@ export const urlIsCompilationAsset = (url) => {
 }
 
 export const getMetaJsonFileUrl = (compileFileUrl) =>
-  generateCompiledFileAssetUrl(compileFileUrl, "meta.json")
+  generateCompilationAssetUrl(compileFileUrl, "meta.json")
 
-export const generateCompiledFileAssetUrl = (compiledFileUrl, assetName) => {
-  return `${compiledFileUrl}__asset__${assetName}`
+export const generateCompilationAssetUrl = (compiledFileUrl, assetName) => {
+  // we want to remove eventual search params from url
+  const origin = urlToOrigin(compiledFileUrl)
+  const pathname = urlToPathname(compiledFileUrl)
+  const compilationAssetUrl = `${origin}${pathname}__asset__${assetName}`
+  return compilationAssetUrl
 }

package/src/internal/compiling/compile-directory/getOrGenerateCompiledFile.js

@@ -1,6 +1,6 @@
+import { timeStart, timeFunction } from "@jsenv/server"
 import { urlToFileSystemPath, readFile } from "@jsenv/filesystem"
 import { createDetailedMessage } from "@jsenv/logger"
-import { timeStart, timeFunction } from "@jsenv/server"
 
 import { validateCache } from "./validateCache.js"
 import { getMetaJsonFileUrl } from "./compile-asset.js"
@@ -14,6 +14,8 @@ export const getOrGenerateCompiledFile = async ({
   projectDirectoryUrl,
   originalFileUrl,
   compiledFileUrl = originalFileUrl,
+  jsenvRemoteDirectory,
+
   compileCacheStrategy,
   compileCacheSourcesValidation,
   compileCacheAssetsValidation,
@@ -61,8 +63,11 @@ export const getOrGenerateCompiledFile = async ({
       const lockTiming = lockTimeEnd()
       const { meta, compileResult, compileResultStatus, timing } =
         await computeCompileReport({
+          projectDirectoryUrl,
           originalFileUrl,
           compiledFileUrl,
+          jsenvRemoteDirectory,
+
           compile,
           compileCacheStrategy,
           compileCacheSourcesValidation,
@@ -89,14 +94,17 @@ export const getOrGenerateCompiledFile = async ({
 }
 
 const computeCompileReport = async ({
+  // projectDirectoryUrl,
+  logger,
   originalFileUrl,
   compiledFileUrl,
+  jsenvRemoteDirectory,
+
   compile,
   compileCacheStrategy,
   compileCacheSourcesValidation,
   compileCacheAssetsValidation,
   request,
-  logger,
 }) => {
   const [readCacheTiming, cacheValidity] = await timeFunction(
     "read cache",
@@ -128,10 +136,41 @@ const computeCompileReport = async ({
     }
 
     const metaIsValid = cacheValidity.meta ? cacheValidity.meta.isValid : false
+
+    const fetchOriginalFile = async () => {
+      // The original file might be behind an http url.
+      // In that case jsenv try first to read file from filesystem
+      // in ".jsenv/.http/" directory. If not found, the url
+      // is fetched and file is written in that ".jsenv/.http/" directory.
+      // After that the only way to re-fetch this ressource is
+      // to delete the content of ".jsenv/.http/"
+      try {
+        const code = await readFile(originalFileUrl)
+        return { code }
+      } catch (e) {
+        // when file is not found and the file is referenced with an http url
+        if (
+          e &&
+          e.code === "ENOENT" &&
+          jsenvRemoteDirectory.isFileUrlForRemoteUrl(originalFileUrl)
+        ) {
+          const responseBodyAsBuffer =
+            await jsenvRemoteDirectory.loadFileUrlFromRemote(
+              originalFileUrl,
+              request,
+            )
+          const code = String(responseBodyAsBuffer)
+          return { code }
+        }
+        throw e
+      }
+    }
+    const { code } = await fetchOriginalFile()
     const [compileTiming, compileResult] = await timeFunction("compile", () =>
       callCompile({
         logger,
         originalFileUrl,
+        code,
         compile,
       }),
     )
@@ -169,12 +208,9 @@ const computeCompileReport = async ({
   }
 }
 
-const callCompile = async ({ logger, originalFileUrl, compile }) => {
+const callCompile = async ({ logger, code, originalFileUrl, compile }) => {
   logger.debug(`compile ${originalFileUrl}`)
-
-  const compileReturnValue = await compile({
-    code: await readFile(originalFileUrl),
-  })
+  const compileReturnValue = await compile({ code })
   if (typeof compileReturnValue !== "object" || compileReturnValue === null) {
     throw new TypeError(
       `compile must return an object, got ${compileReturnValue}`,
@@ -200,7 +236,6 @@ const callCompile = async ({ logger, originalFileUrl, compile }) => {
       `compile must return a compiledSource string, got ${compiledSource}`,
     )
   }
-
   return {
     contentType,
     compiledSource,

package/src/internal/compiling/compile-directory/updateMeta.js

@@ -30,14 +30,10 @@ export const updateMeta = async ({
   const promises = []
   if (isNew || isUpdated) {
     // ensure source that does not leads to concrete files are not capable to invalidate the cache
-    const sourcesToRemove = []
-    sources.forEach((sourceFileUrl) => {
-      const sourceFileExists = testFilePresence(sourceFileUrl)
-      if (sourceFileExists) {
-        return
-      }
-
-      sourcesToRemove.push(sourceFileUrl)
+    const sourcesToRemove = sources.filter((sourceFileUrl) => {
+      return (
+        sourceFileUrl.startsWith("file://") && !testFilePresence(sourceFileUrl)
+      )
     })
     const sourceNotFoundCount = sourcesToRemove.length
     if (sourceNotFoundCount > 0) {

package/src/internal/compiling/compile-directory/validateCache.js

@@ -1,7 +1,6 @@
-import { resolveUrl, bufferToEtag } from "@jsenv/filesystem"
-
 import { fileURLToPath } from "node:url"
 import { readFileSync, statSync } from "node:fs"
+import { resolveUrl, bufferToEtag } from "@jsenv/filesystem"
 
 export const validateCache = async ({
   compiledFileUrl,

package/src/internal/compiling/compileFile.js

@@ -14,10 +14,13 @@ export const compileFile = async ({
   logger,
 
   projectDirectoryUrl,
+  jsenvRemoteDirectory,
   originalFileUrl,
   compiledFileUrl,
+
   projectFileRequestedCallback = () => {},
   request,
+  responseHeaders = {},
   pushResponse,
   importmapInfos,
   compile,
@@ -35,6 +38,11 @@ export const compileFile = async ({
     )
   }
 
+  projectFileRequestedCallback(
+    urlToRelativeUrl(originalFileUrl, projectDirectoryUrl),
+    request,
+  )
+
   const clientCacheDisabled = request.headers["cache-control"] === "no-cache"
 
   try {
@@ -44,6 +52,8 @@ export const compileFile = async ({
         projectDirectoryUrl,
         originalFileUrl,
         compiledFileUrl,
+        jsenvRemoteDirectory,
+
         request,
         compileCacheStrategy,
         compileCacheSourcesValidation,
@@ -78,13 +88,15 @@ export const compileFile = async ({
       compileResult.compiledMtime = Date.now()
     }
 
-    const {
-      contentType,
-      compiledEtag,
-      compiledMtime,
-      compiledSource,
-      responseHeaders = {},
-    } = compileResult
+    const { contentType, compiledEtag, compiledMtime, compiledSource } =
+      compileResult
+
+    if (compileResult.responseHeaders) {
+      responseHeaders = {
+        ...responseHeaders,
+        ...compileResult.responseHeaders,
+      }
+    }
 
     if (compileResultStatus !== "cached" && compileCacheStrategy !== "none") {
       // we MUST await updateMeta otherwise we might get 404
@@ -207,7 +219,6 @@ export const compileFile = async ({
       }
       // on the correspondig file
       const json = JSON.stringify(data)
-
       return {
         status: 500,
         statusText: "parse error",
@@ -219,13 +230,14 @@ export const compileFile = async ({
         body: json,
       }
     }
-
+    if (error && error.asResponse) {
+      return error.asResponse()
+    }
     if (error && error.statusText === "Unexpected directory operation") {
       return {
         status: 403,
       }
     }
-
     return convertFileSystemErrorToResponseProperties(error)
   }
 }

package/src/internal/compiling/createCompiledFileService.js

@@ -38,7 +38,9 @@ export const createCompiledFileService = ({
   logger,
 
   projectDirectoryUrl,
+  jsenvDirectoryRelativeUrl,
   outDirectoryRelativeUrl,
+  jsenvRemoteDirectory,
 
   runtimeSupport,
   babelPluginMap,
@@ -89,12 +91,9 @@ export const createCompiledFileService = ({
 
   const importmapInfos = {}
 
-  return (request, { pushResponse, redirectRequest }) => {
+  return async (request, { pushResponse, redirectRequest }) => {
     const { origin, ressource } = request
-    // we use "ressourceToPathname" to remove eventual query param from the url
-    // Without this a pattern like "**/*.js" would not match "file.js?t=1"
-    // This would result in file not being compiled when they should
-    const requestUrl = `${origin}${ressourceToPathname(ressource)}`
+    const requestUrl = `${origin}${ressource}`
 
     const requestCompileInfo = serverUrlToCompileInfo(requestUrl, {
       outDirectoryRelativeUrl,
@@ -140,8 +139,6 @@ export const createCompiledFileService = ({
     }
 
     const originalFileRelativeUrl = afterCompileId
-    projectFileRequestedCallback(originalFileRelativeUrl, request)
-
     const originalFileUrl = `${projectDirectoryUrl}${originalFileRelativeUrl}`
     const compileDirectoryRelativeUrl = `${outDirectoryRelativeUrl}${compileId}/`
     const compileDirectoryUrl = resolveDirectoryUrl(
@@ -152,23 +149,22 @@ export const createCompiledFileService = ({
       originalFileRelativeUrl,
       compileDirectoryUrl,
     )
-
     const compiler = getCompiler({ originalFileUrl, compileMeta })
     // no compiler -> serve original file
-    // we don't redirect otherwise it complexify ressource tracking
-    // and url resolution
+    // we redirect "internally" (we dont send 304 to the browser)
+    // to keep ressource tracking and url resolution simple
     if (!compiler) {
       return redirectRequest({
         pathname: `/${originalFileRelativeUrl}`,
       })
     }
-
     // compile this if needed
     const compileResponsePromise = compileFile({
       compileServerOperation,
       logger,
 
       projectDirectoryUrl,
+      jsenvRemoteDirectory,
       originalFileUrl,
       compiledFileUrl,
 
@@ -181,14 +177,16 @@ export const createCompiledFileService = ({
         return compiler({
           logger,
 
-          code,
-          url: originalFileUrl,
-          compiledUrl: compiledFileUrl,
           projectDirectoryUrl,
+          jsenvRemoteDirectory,
           compileServerOrigin: request.origin,
+          jsenvDirectoryRelativeUrl,
           outDirectoryRelativeUrl,
-          compileId,
+          url: originalFileUrl,
+          compiledUrl: compiledFileUrl,
           request,
+
+          compileId,
           babelPluginMap: shakeBabelPluginMap({
             babelPluginMap,
             missingFeatureNames: groupMap[compileId].missingFeatureNames,
@@ -204,6 +202,7 @@ export const createCompiledFileService = ({
           topLevelAwait,
           prependSystemJs,
 
+          code,
           sourcemapMethod,
           sourcemapExcludeSources,
           jsenvEventSourceClientInjection,
@@ -238,6 +237,14 @@ const canAvoidSystemJs = ({
 }
 
 const getCompiler = ({ originalFileUrl, compileMeta }) => {
+  // we remove eventual query param from the url
+  // Without this a pattern like "**/*.js" would not match "file.js?t=1"
+  // This would result in file not being compiled when they should
+  // Ideally we would do a first pass with the query param and a second without
+  const urlObject = new URL(originalFileUrl)
+  urlObject.search = ""
+  originalFileUrl = urlObject.href
+
   const { jsenvCompiler, customCompiler } = urlToMeta({
     url: originalFileUrl,
     structuredMetaMap: compileMeta,
@@ -307,12 +314,3 @@ const contentTypeExtensions = {
   "application/importmap+json": ".importmap",
   // "text/css": ".css",
 }
-
-const ressourceToPathname = (ressource) => {
-  const searchSeparatorIndex = ressource.indexOf("?")
-  const pathname =
-    searchSeparatorIndex === -1
-      ? ressource
-      : ressource.slice(0, searchSeparatorIndex)
-  return pathname
-}