@jsenv/core 27.0.0-alpha.8 → 27.0.0-alpha.80

package/src/omega/omega_server.js

@@ -8,14 +8,13 @@ import {
 } from "@jsenv/server"
 import { convertFileSystemErrorToResponseProperties } from "@jsenv/server/src/internal/convertFileSystemErrorToResponseProperties.js"
 import { createCallbackListNotifiedOnce } from "@jsenv/abort"
-import { loggerToLogLevel } from "@jsenv/logger"
 
 import { createFileService } from "./server/file_service.js"
 
 export const startOmegaServer = async ({
   signal,
   handleSIGINT,
-  logger,
+  logLevel,
   protocol = "http",
   http2 = protocol === "https",
   privateKey,
@@ -48,7 +47,7 @@ export const startOmegaServer = async ({
     stopOnSIGINT: handleSIGINT,
     stopOnInternalError: false,
     keepProcessAlive,
-    logLevel: loggerToLogLevel(logger),
+    logLevel,
     startLog: false,
 
     protocol,

package/src/omega/server/file_service.js

@@ -3,7 +3,7 @@ import {
   serveDirectory,
   composeTwoResponses,
 } from "@jsenv/server"
-import { urlIsInsideOf, moveUrl } from "@jsenv/filesystem"
+import { urlIsInsideOf, moveUrl } from "@jsenv/urls"
 
 import { parseUserAgentHeader } from "./user_agent.js"
 
@@ -20,6 +20,11 @@ export const createFileService = ({
     urlGraph,
     scenario,
   }
+  const augmentResponseContext = {
+    rootDirectoryUrl,
+    urlGraph,
+    scenario,
+  }
 
   const getResponse = async (request) => {
     // serve file inside ".jsenv" directory
@@ -39,43 +44,87 @@ export const createFileService = ({
     if (responseFromPlugin) {
       return responseFromPlugin
     }
-    const { runtimeName, runtimeVersion } = parseUserAgentHeader(
-      request.headers["user-agent"],
-    )
-    const runtimeSupport = {
-      [runtimeName]: runtimeVersion,
+    let reference
+    const parentUrl = inferParentFromRequest(request, rootDirectoryUrl)
+    if (parentUrl) {
+      reference = urlGraph.inferReference(request.ressource, parentUrl)
+    }
+    if (!reference) {
+      const entryPoint = kitchen.prepareEntryPoint({
+        trace: parentUrl || rootDirectoryUrl,
+        parentUrl: parentUrl || rootDirectoryUrl,
+        type: "entry_point",
+        specifier: request.ressource,
+      })
+      reference = entryPoint[0]
+    }
+    const urlInfo = urlGraph.reuseOrCreateUrlInfo(reference.url)
+
+    const ifNoneMatch = request.headers["if-none-match"]
+    if (ifNoneMatch && urlInfo.contentEtag === ifNoneMatch) {
+      return {
+        status: 304,
+        headers: {
+          "cache-control": `private,max-age=0,must-revalidate`,
+          ...urlInfo.headers,
+        },
+      }
     }
-    const reference = kitchen.createReference({
-      parentUrl: inferParentFromRequest(request, rootDirectoryUrl),
-      type: "entry_point",
-      specifier: request.ressource,
-    })
-    const requestedUrlInfo = kitchen.resolveReference(reference)
-    const referenceFromGraph = urlGraph.inferReference(
-      reference.url,
-      reference.parentUrl,
-    )
     try {
-      await kitchen.cook({
-        reference: referenceFromGraph || reference,
-        urlInfo: requestedUrlInfo,
-        outDirectoryUrl: `${rootDirectoryUrl}.jsenv/${scenario}/${runtimeName}@${runtimeVersion}/`,
-        runtimeSupport,
+      // urlInfo objects are reused, they must be "reset" before cooking them again
+      if (
+        urlInfo.contentEtag &&
+        !urlInfo.isInline &&
+        urlInfo.type !== "sourcemap"
+      ) {
+        urlInfo.sourcemap = null
+        urlInfo.sourcemapReference = null
+        urlInfo.content = null
+        urlInfo.originalContent = null
+        urlInfo.type = null
+        urlInfo.subtype = null
+        urlInfo.timing = {}
+      }
+      const { runtimeName, runtimeVersion } = parseUserAgentHeader(
+        request.headers["user-agent"],
+      )
+      await kitchen.cook(urlInfo, {
+        request,
+        reference,
+        clientRuntimeCompat: {
+          [runtimeName]: runtimeVersion,
+        },
+        outDirectoryUrl:
+          scenario === "dev"
+            ? `${rootDirectoryUrl}.jsenv/${runtimeName}@${runtimeVersion}/`
+            : `${rootDirectoryUrl}.jsenv/${scenario}/${runtimeName}@${runtimeVersion}/`,
       })
-      const { response, contentType, content } = requestedUrlInfo
+      let { response } = urlInfo
       if (response) {
         return response
       }
-      return {
+      response = {
         url: reference.url,
         status: 200,
         headers: {
-          "content-type": contentType,
-          "content-length": Buffer.byteLength(content),
+          "content-length": Buffer.byteLength(urlInfo.content),
           "cache-control": `private,max-age=0,must-revalidate`,
+          "eTag": urlInfo.contentEtag,
+          ...urlInfo.headers,
+          "content-type": urlInfo.contentType,
         },
-        body: content,
+        body: urlInfo.content,
+        timing: urlInfo.timing,
       }
+      kitchen.pluginController.callHooks(
+        "augmentResponse",
+        { reference, urlInfo },
+        augmentResponseContext,
+        (returnValue) => {
+          response = composeTwoResponses(response, returnValue)
+        },
+      )
+      return response
     } catch (e) {
       const code = e.code
       if (code === "PARSE_ERROR") {
@@ -86,14 +135,14 @@ export const createFileService = ({
           statusText: e.reason,
           statusMessage: e.message,
           headers: {
-            "content-type": requestedUrlInfo.contentType,
-            "content-length": Buffer.byteLength(requestedUrlInfo.content),
+            "content-type": urlInfo.contentType,
+            "content-length": Buffer.byteLength(urlInfo.content),
             "cache-control": "no-store",
           },
-          body: requestedUrlInfo.content,
+          body: urlInfo.content,
         }
       }
-      if (code === "EISDIR") {
+      if (code === "DIRECTORY_REFERENCE_NOT_ALLOWED") {
         return serveDirectory(reference.url, {
           headers: {
             accept: "text/html",
@@ -127,16 +176,6 @@ export const createFileService = ({
   }
   return async (request) => {
     let response = await getResponse(request)
-    if (response.url) {
-      kitchen.pluginController.callHooks(
-        "augmentResponse",
-        response,
-        {},
-        (returnValue) => {
-          response = composeTwoResponses(response, returnValue)
-        },
-      )
-    }
     return response
   }
 }
@@ -144,7 +183,15 @@ export const createFileService = ({
 const inferParentFromRequest = (request, rootDirectoryUrl) => {
   const { referer } = request.headers
   if (!referer) {
-    return rootDirectoryUrl
+    return null
+  }
+  const refererUrlObject = new URL(referer)
+  refererUrlObject.searchParams.delete("hmr")
+  refererUrlObject.searchParams.delete("v")
+  const { pathname, search } = refererUrlObject
+  if (pathname.startsWith("/@fs/")) {
+    const fsRootRelativeUrl = pathname.slice("/@fs/".length)
+    return `file:///${fsRootRelativeUrl}${search}`
   }
   return moveUrl({
     url: referer,

package/src/omega/server/user_agent.js

@@ -1,8 +1,7 @@
-import { createRequire } from "node:module"
+import { memoizeByFirstArgument } from "@jsenv/utils/memoize/memoize_by_first_argument.js"
+import { requireFromJsenv } from "@jsenv/core/src/require_from_jsenv.js"
 
-const require = createRequire(import.meta.url)
-
-export const parseUserAgentHeader = (userAgent) => {
+export const parseUserAgentHeader = memoizeByFirstArgument((userAgent) => {
   if (userAgent.includes("node-fetch/")) {
     // it's not really node and conceptually we can't assume the node version
     // but good enough for now
@@ -11,7 +10,7 @@ export const parseUserAgentHeader = (userAgent) => {
       runtimeVersion: process.version.slice(1),
     }
   }
-  const UA = require("@financial-times/polyfill-useragent-normaliser")
+  const UA = requireFromJsenv("@financial-times/polyfill-useragent-normaliser")
   const { ua } = new UA(userAgent)
   const { family, major, minor, patch } = ua
   return {
@@ -19,4 +18,4 @@ export const parseUserAgentHeader = (userAgent) => {
     runtimeVersion:
       family === "Other" ? "unknown" : `${major}.${minor}${patch}`,
   }
-}
+})

package/src/omega/url_graph/url_graph_load.js

@@ -1,62 +1,69 @@
 import { ensureEmptyDirectory } from "@jsenv/filesystem"
 
 export const loadUrlGraph = async ({
+  operation,
   urlGraph,
   kitchen,
   startLoading,
+  writeGeneratedFiles,
   outDirectoryUrl,
-  runtimeSupport,
+  clientRuntimeCompat,
 }) => {
-  if (outDirectoryUrl) {
+  if (writeGeneratedFiles && outDirectoryUrl) {
     await ensureEmptyDirectory(outDirectoryUrl)
   }
   const promises = []
-  const cook = ({ urlInfo, ...rest }) => {
-    const promiseFromData = urlInfo.data.promise
+  const promiseMap = new Map()
+  const cook = (urlInfo, context) => {
+    const promiseFromData = promiseMap.get(urlInfo)
     if (promiseFromData) return promiseFromData
-    const promise = _cook({
-      urlInfo,
+    const promise = _cook(urlInfo, {
       outDirectoryUrl,
-      runtimeSupport,
-      ...rest,
+      clientRuntimeCompat,
+      ...context,
     })
     promises.push(promise)
-    urlInfo.data.promise = promise
+    promiseMap.set(urlInfo, promise)
     return promise
   }
-  const _cook = async ({ urlInfo, ...rest }) => {
-    await kitchen.cook({
-      urlInfo,
+  const _cook = async (urlInfo, context) => {
+    await kitchen.cook(urlInfo, {
       cookDuringCook: cook,
-      ...rest,
+      ...context,
     })
     const { references } = urlInfo
     references.forEach((reference) => {
-      cook({
-        reference,
-        urlInfo: urlGraph.getUrlInfo(reference.url),
-      })
+      // we don't cook ressource hints
+      // because they might refer to ressource that will be modified during build
+      // It also means something else have to reference that url in order to cook it
+      // so that the preload is deleted by "resync_ressource_hints.js" otherwise
+      if (reference.isRessourceHint) {
+        return
+      }
+      // we use reference.generatedUrl to mimic what a browser would do:
+      // do a fetch to the specifier as found in the file
+      const referencedUrlInfo = urlGraph.reuseOrCreateUrlInfo(
+        reference.generatedUrl,
+      )
+      cook(referencedUrlInfo, { reference })
     })
   }
   startLoading(
     ({ trace, parentUrl = kitchen.rootDirectoryUrl, type, specifier }) => {
-      const entryReference = kitchen.createReference({
+      const [entryReference, entryUrlInfo] = kitchen.prepareEntryPoint({
         trace,
         parentUrl,
         type,
         specifier,
       })
-      const entryUrlInfo = kitchen.resolveReference(entryReference)
       entryUrlInfo.data.isEntryPoint = true
-      cook({
-        reference: entryReference,
-        urlInfo: entryUrlInfo,
-      })
+      cook(entryUrlInfo, { reference: entryReference })
       return [entryReference, entryUrlInfo]
     },
   )
 
   const waitAll = async () => {
+    operation.throwIfAborted()
     if (promises.length === 0) {
       return
     }
@@ -66,4 +73,5 @@ export const loadUrlGraph = async ({
     await waitAll()
   }
   await waitAll()
+  promiseMap.clear()
 }

package/src/omega/url_graph/url_graph_report.js

@@ -1,37 +1,33 @@
-import { ANSI } from "@jsenv/log"
-
-import { byteAsFileSize } from "@jsenv/utils/logs/size_log.js"
+import { ANSI, byteAsFileSize, distributePercentages } from "@jsenv/log"
 
 export const createUrlGraphSummary = (
   urlGraph,
   { title = "graph summary" } = {},
 ) => {
   const graphReport = createUrlGraphReport(urlGraph)
-  const totalLabel = `Total`
   return `--- ${title} ---  
 ${createRepartitionMessage(graphReport)}
-${ANSI.color(totalLabel, ANSI.GREY)} ${
-    graphReport.total.count
-  } (${byteAsFileSize(graphReport.total.size)})
 --------------------`
 }
 
 const createUrlGraphReport = (urlGraph) => {
   const { urlInfos } = urlGraph
   const countGroups = {
+    sourcemaps: 0,
     html: 0,
     css: 0,
     js: 0,
-    assets: 0,
-    sourcemaps: 0,
+    json: 0,
+    other: 0,
     total: 0,
   }
   const sizeGroups = {
+    sourcemaps: 0,
     html: 0,
     css: 0,
     js: 0,
-    sourcemaps: 0,
-    assets: 0,
+    json: 0,
+    other: 0,
     total: 0,
   }
   Object.keys(urlInfos).forEach((url) => {
@@ -39,8 +35,10 @@ const createUrlGraphReport = (urlGraph) => {
       return
     }
     const urlInfo = urlInfos[url]
-    // ignore inline files, they are already taken into account in the file where they appear
-    if (urlInfo.isInline) {
+    // ignore:
+    // - inline files: they are already taken into account in the file where they appear
+    // - ignored files: we don't know their content
+    if (urlInfo.isInline || !urlInfo.shouldHandle) {
       return
     }
     // file loaded via import assertion are already inside the graph
@@ -49,9 +47,9 @@ const createUrlGraphReport = (urlGraph) => {
     // and only the js module remain (likely bundled)
     const urlObject = new URL(urlInfo.url)
     if (
-      urlObject.searchParams.has("json_module") ||
-      urlObject.searchParams.has("css_module") ||
-      urlObject.searchParams.has("text_module")
+      urlObject.searchParams.has("as_json_module") ||
+      urlObject.searchParams.has("as_css_module") ||
+      urlObject.searchParams.has("as_text_module")
     ) {
       return
     }
@@ -79,17 +77,63 @@ const createUrlGraphReport = (urlGraph) => {
       sizeGroups.js += urlContentSize
       return
     }
-    countGroups.assets++
-    sizeGroups.assets += urlContentSize
+    if (category === "json") {
+      countGroups.json++
+      sizeGroups.json += urlContentSize
+      return
+    }
+    countGroups.other++
+    sizeGroups.other += urlContentSize
     return
   })
+
+  const sizesToDistribute = {}
+  Object.keys(sizeGroups).forEach((groupName) => {
+    if (groupName !== "sourcemaps" && groupName !== "total") {
+      sizesToDistribute[groupName] = sizeGroups[groupName]
+    }
+  })
+  const percentageGroups = distributePercentages(sizesToDistribute)
+
   return {
-    html: { count: countGroups.html, size: sizeGroups.html },
-    css: { count: countGroups.css, size: sizeGroups.css },
-    js: { count: countGroups.js, size: sizeGroups.js },
-    sourcemaps: { count: countGroups.sourcemaps, size: sizeGroups.sourcemaps },
-    assets: { count: countGroups.assets, size: sizeGroups.assets },
-    total: { count: countGroups.total, size: sizeGroups.total },
+    // sourcemaps are special, there size are ignored
+    // so there is no "percentage" associated
+    sourcemaps: {
+      count: countGroups.sourcemaps,
+      size: sizeGroups.sourcemaps,
+      percentage: undefined,
+    },
+
+    html: {
+      count: countGroups.html,
+      size: sizeGroups.html,
+      percentage: percentageGroups.html,
+    },
+    css: {
+      count: countGroups.css,
+      size: sizeGroups.css,
+      percentage: percentageGroups.css,
+    },
+    js: {
+      count: countGroups.js,
+      size: sizeGroups.js,
+      percentage: percentageGroups.js,
+    },
+    json: {
+      count: countGroups.json,
+      size: sizeGroups.json,
+      percentage: percentageGroups.json,
+    },
+    other: {
+      count: countGroups.other,
+      size: sizeGroups.other,
+      percentage: percentageGroups.other,
+    },
+    total: {
+      count: countGroups.total,
+      size: sizeGroups.total,
+      percentage: 100,
+    },
   }
 }
 
@@ -106,32 +150,22 @@ const determineCategory = (urlInfo) => {
   if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
     return "js"
   }
-  return "assets"
+  if (urlInfo.type === "json") {
+    return "json"
+  }
+  return "other"
 }
 
-const createRepartitionMessage = ({ html, css, js, assets }) => {
-  const parts = []
-  if (html.count) {
-    parts.push(
-      `${ANSI.color(`html:`, ANSI.GREY)} ${html.count} (${byteAsFileSize(
-        html.size,
-      )})`,
-    )
-  }
-  if (css.count) {
+const createRepartitionMessage = ({ html, css, js, json, other, total }) => {
+  const addPart = (name, { count, size, percentage }) => {
     parts.push(
-      `${ANSI.color(`css:`, ANSI.GREY)} ${css.count} (${byteAsFileSize(
-        css.size,
-      )})`,
-    )
-  }
-  if (js.count) {
-    parts.push(
-      `${ANSI.color(`js:`, ANSI.GREY)} ${js.count} (${byteAsFileSize(
-        js.size,
-      )})`,
+      `${ANSI.color(`${name}:`, ANSI.GREY)} ${count} (${byteAsFileSize(
+        size,
+      )} / ${percentage} %)`,
     )
   }
+
+  const parts = []
   // if (sourcemaps.count) {
   //   parts.push(
   //     `${ANSI.color(`sourcemaps:`, ANSI.GREY)} ${
@@ -139,13 +173,22 @@ const createRepartitionMessage = ({ html, css, js, assets }) => {
   //     } (${byteAsFileSize(sourcemaps.size)})`,
   //   )
   // }
-  if (assets.count) {
-    parts.push(
-      `${ANSI.color(`assets:`, ANSI.GREY)} ${assets.count} (${byteAsFileSize(
-        assets.size,
-      )})`,
-    )
+  if (html.count) {
+    addPart("html ", html)
+  }
+  if (css.count) {
+    addPart("css  ", css)
+  }
+  if (js.count) {
+    addPart("js   ", js)
+  }
+  if (json.count) {
+    addPart("json ", json)
+  }
+  if (other.count) {
+    addPart("other", other)
   }
+  addPart("total", total)
   return `- ${parts.join(`
 - `)}`
 }

package/src/omega/url_graph/url_info_transformations.js

@@ -1,3 +1,6 @@
+import { bufferToEtag } from "@jsenv/filesystem"
+import { urlToRelativeUrl } from "@jsenv/urls"
+
 import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
 import {
   SOURCEMAP,
@@ -8,7 +11,8 @@ import {
 export const createUrlInfoTransformer = ({
   logger,
   sourcemaps,
-  sourcemapsSources,
+  sourcemapsSourcesContent,
+  sourcemapsRelativeSources,
   urlGraph,
   injectSourcemapPlaceholder,
   foundSourcemap,
@@ -23,17 +27,22 @@ export const createUrlInfoTransformer = ({
       // for inline content (<script> insdide html)
       // chrome won't be able to fetch the file as it does not exists
       // so sourcemap must contain sources
-      urlInfo.isInline || sourcemapsSources
+      sourcemapsSourcesContent ||
+      urlInfo.isInline ||
+      (sourcemap.sources &&
+        sourcemap.sources.some(
+          (source) => !source || !source.startsWith("file:"),
+        ))
     if (sourcemap.sources && sourcemap.sources.length > 1) {
       sourcemap.sources = sourcemap.sources.map(
-        (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
+        (source) => new URL(source, urlInfo.originalUrl).href,
       )
       if (!wantSourcesContent) {
         sourcemap.sourcesContent = undefined
       }
       return sourcemap
     }
-    sourcemap.sources = [urlInfo.data.rawUrl || urlInfo.url]
+    sourcemap.sources = [urlInfo.originalUrl]
     sourcemap.sourcesContent = [urlInfo.originalContent]
     if (!wantSourcesContent) {
       sourcemap.sourcesContent = undefined
@@ -66,6 +75,7 @@ export const createUrlInfoTransformer = ({
 
     // already loaded during "load" hook (happens during build)
     if (urlInfo.sourcemap) {
+      urlInfo.sourcemap = normalizeSourcemap(urlInfo, urlInfo.sourcemap)
       return
     }
     // check for existing sourcemap for this content
@@ -78,17 +88,15 @@ export const createUrlInfoTransformer = ({
       const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
         urlInfo,
         type,
-        line,
-        column,
         specifier,
+        specifierLine: line,
+        specifierColumn: column,
       })
       try {
-        await context.cook({
-          reference: sourcemapReference,
-          urlInfo: sourcemapUrlInfo,
-        })
-        const sourcemap = JSON.parse(sourcemapUrlInfo.content)
-        urlInfo.sourcemap = normalizeSourcemap(urlInfo, sourcemap)
+        await context.cook(sourcemapUrlInfo, { reference: sourcemapReference })
+        const sourcemapRaw = JSON.parse(sourcemapUrlInfo.content)
+        const sourcemap = normalizeSourcemap(urlInfo, sourcemapRaw)
+        urlInfo.sourcemap = sourcemap
       } catch (e) {
         logger.error(`Error while handling existing sourcemap: ${e.message}`)
         return
@@ -137,20 +145,32 @@ export const createUrlInfoTransformer = ({
       const sourcemapReference = urlInfo.sourcemapReference
       const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
       sourcemapUrlInfo.contentType = "application/json"
-      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      const sourcemap = urlInfo.sourcemap
+      if (sourcemapsRelativeSources) {
+        sourcemap.sources = sourcemap.sources.map((source) => {
+          const sourceRelative = urlToRelativeUrl(source, urlInfo.url)
+          return sourceRelative || "."
+        })
+      }
+      sourcemapUrlInfo.content = JSON.stringify(sourcemap, null, "  ")
       if (sourcemaps === "inline") {
-        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
-          urlInfo.sourcemap,
-        )
+        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(sourcemap)
       }
       if (sourcemaps === "file" || sourcemaps === "inline") {
         urlInfo.content = SOURCEMAP.writeComment({
           contentType: urlInfo.contentType,
           content: urlInfo.content,
-          specifier: sourcemapReference.generatedSpecifier,
+          specifier:
+            sourcemaps === "file" && sourcemapsRelativeSources
+              ? urlToRelativeUrl(sourcemapReference.url, urlInfo.url)
+              : sourcemapReference.generatedSpecifier,
         })
       }
+    } else if (urlInfo.sourcemapReference) {
+      // in the end we don't use the sourcemap placeholder
+      urlGraph.deleteUrlInfo(urlInfo.sourcemapReference.url)
     }
+    urlInfo.contentEtag = bufferToEtag(Buffer.from(urlInfo.content))
   }
 
   return {