@jsenv/core 27.0.0-alpha.6 → 27.0.0-alpha.60

package/src/omega/omega_server.js

@@ -8,14 +8,13 @@ import {
 } from "@jsenv/server"
 import { convertFileSystemErrorToResponseProperties } from "@jsenv/server/src/internal/convertFileSystemErrorToResponseProperties.js"
 import { createCallbackListNotifiedOnce } from "@jsenv/abort"
-import { loggerToLogLevel } from "@jsenv/logger"
 
 import { createFileService } from "./server/file_service.js"
 
 export const startOmegaServer = async ({
   signal,
   handleSIGINT,
-  logger,
+  logLevel,
   protocol = "http",
   http2 = protocol === "https",
   privateKey,
@@ -48,7 +47,7 @@ export const startOmegaServer = async ({
     stopOnSIGINT: handleSIGINT,
     stopOnInternalError: false,
     keepProcessAlive,
-    logLevel: loggerToLogLevel(logger),
+    logLevel,
     startLog: false,
 
     protocol,

package/src/omega/server/file_service.js

@@ -20,6 +20,11 @@ export const createFileService = ({
     urlGraph,
     scenario,
   }
+  const augmentResponseContext = {
+    rootDirectoryUrl,
+    urlGraph,
+    scenario,
+  }
 
   const getResponse = async (request) => {
     // serve file inside ".jsenv" directory
@@ -39,43 +44,75 @@ export const createFileService = ({
     if (responseFromPlugin) {
       return responseFromPlugin
     }
-    const { runtimeName, runtimeVersion } = parseUserAgentHeader(
-      request.headers["user-agent"],
-    )
-    const runtimeSupport = {
-      [runtimeName]: runtimeVersion,
-    }
-    const reference = kitchen.createReference({
+    const [reference, urlInfo] = kitchen.prepareEntryPoint({
       parentUrl: inferParentFromRequest(request, rootDirectoryUrl),
       type: "entry_point",
       specifier: request.ressource,
     })
-    const requestedUrlInfo = kitchen.resolveReference(reference)
+    const ifNoneMatch = request.headers["if-none-match"]
+    if (ifNoneMatch && urlInfo.contentEtag === ifNoneMatch) {
+      return {
+        status: 304,
+        headers: {
+          "cache-control": `private,max-age=0,must-revalidate`,
+        },
+      }
+    }
     const referenceFromGraph = urlGraph.inferReference(
       reference.url,
       reference.parentUrl,
     )
     try {
+      // urlInfo objects are reused, they must be "reset" before cooking them again
+      if (
+        urlInfo.contentEtag &&
+        !urlInfo.isInline &&
+        urlInfo.type !== "sourcemap"
+      ) {
+        urlInfo.sourcemap = null
+        urlInfo.sourcemapReference = null
+        urlInfo.content = null
+        urlInfo.originalContent = null
+        urlInfo.type = null
+        urlInfo.subtype = null
+        urlInfo.timing = {}
+      }
+      const { runtimeName, runtimeVersion } = parseUserAgentHeader(
+        request.headers["user-agent"],
+      )
       await kitchen.cook({
         reference: referenceFromGraph || reference,
-        urlInfo: requestedUrlInfo,
+        urlInfo,
         outDirectoryUrl: `${rootDirectoryUrl}.jsenv/${scenario}/${runtimeName}@${runtimeVersion}/`,
-        runtimeSupport,
+        clientRuntimeCompat: {
+          [runtimeName]: runtimeVersion,
+        },
       })
-      const { response, contentType, content } = requestedUrlInfo
+      let { response, contentType, content, contentEtag } = urlInfo
       if (response) {
         return response
       }
-      return {
+      response = {
         url: reference.url,
         status: 200,
         headers: {
           "content-type": contentType,
           "content-length": Buffer.byteLength(content),
           "cache-control": `private,max-age=0,must-revalidate`,
+          "eTag": contentEtag,
         },
         body: content,
+        timing: urlInfo.timing,
       }
+      kitchen.pluginController.callHooks(
+        "augmentResponse",
+        { reference, urlInfo },
+        augmentResponseContext,
+        (returnValue) => {
+          response = composeTwoResponses(response, returnValue)
+        },
+      )
+      return response
     } catch (e) {
       const code = e.code
       if (code === "PARSE_ERROR") {
@@ -86,11 +123,11 @@ export const createFileService = ({
           statusText: e.reason,
           statusMessage: e.message,
           headers: {
-            "content-type": requestedUrlInfo.contentType,
-            "content-length": Buffer.byteLength(requestedUrlInfo.content),
+            "content-type": urlInfo.contentType,
+            "content-length": Buffer.byteLength(urlInfo.content),
             "cache-control": "no-store",
           },
-          body: requestedUrlInfo.content,
+          body: urlInfo.content,
         }
       }
       if (code === "EISDIR") {
@@ -127,16 +164,7 @@ export const createFileService = ({
   }
   return async (request) => {
     let response = await getResponse(request)
-    if (response.url) {
-      kitchen.pluginController.callHooks(
-        "augmentResponse",
-        response,
-        {},
-        (returnValue) => {
-          response = composeTwoResponses(response, returnValue)
-        },
-      )
-    }
+
     return response
   }
 }

package/src/omega/server/user_agent.js

@@ -1,8 +1,10 @@
 import { createRequire } from "node:module"
 
+import { memoizeByFirstArgument } from "@jsenv/utils/memoize/memoize_by_first_argument.js"
+
 const require = createRequire(import.meta.url)
 
-export const parseUserAgentHeader = (userAgent) => {
+export const parseUserAgentHeader = memoizeByFirstArgument((userAgent) => {
   if (userAgent.includes("node-fetch/")) {
     // it's not really node and conceptually we can't assume the node version
     // but good enough for now
@@ -19,4 +21,4 @@ export const parseUserAgentHeader = (userAgent) => {
     runtimeVersion:
       family === "Other" ? "unknown" : `${major}.${minor}${patch}`,
   }
-}
+})

package/src/omega/url_graph/url_graph_load.js

@@ -1,27 +1,29 @@
 import { ensureEmptyDirectory } from "@jsenv/filesystem"
 
 export const loadUrlGraph = async ({
+  operation,
   urlGraph,
   kitchen,
   startLoading,
   outDirectoryUrl,
-  runtimeSupport,
+  clientRuntimeCompat,
 }) => {
   if (outDirectoryUrl) {
     await ensureEmptyDirectory(outDirectoryUrl)
   }
   const promises = []
+  const promiseMap = new Map()
   const cook = ({ urlInfo, ...rest }) => {
-    const promiseFromData = urlInfo.data.promise
+    const promiseFromData = promiseMap.get(urlInfo)
     if (promiseFromData) return promiseFromData
     const promise = _cook({
       urlInfo,
       outDirectoryUrl,
-      runtimeSupport,
+      clientRuntimeCompat,
       ...rest,
     })
     promises.push(promise)
-    urlInfo.data.promise = promise
+    promiseMap.set(urlInfo, promise)
     return promise
   }
   const _cook = async ({ urlInfo, ...rest }) => {
@@ -32,21 +34,32 @@ export const loadUrlGraph = async ({
     })
     const { references } = urlInfo
     references.forEach((reference) => {
+      // we don't cook ressource hints
+      // because they might refer to ressource that will be modified during build
+      // It also means something else have to reference that url in order to cook it
+      // so that the preload is deleted by "resync_ressource_hints.js" otherwise
+      if (reference.isRessourceHint) {
+        return
+      }
+      // we use reference.generatedUrl to mimic what a browser would do:
+      // do a fetch to the specifier as found in the file
+      const referencedUrlInfo = urlGraph.reuseOrCreateUrlInfo(
+        reference.generatedUrl,
+      )
       cook({
         reference,
-        urlInfo: urlGraph.getUrlInfo(reference.url),
+        urlInfo: referencedUrlInfo,
       })
     })
   }
   startLoading(
     ({ trace, parentUrl = kitchen.rootDirectoryUrl, type, specifier }) => {
-      const entryReference = kitchen.createReference({
+      const [entryReference, entryUrlInfo] = kitchen.prepareEntryPoint({
         trace,
         parentUrl,
         type,
         specifier,
       })
-      const entryUrlInfo = kitchen.resolveReference(entryReference)
       entryUrlInfo.data.isEntryPoint = true
       cook({
         reference: entryReference,
@@ -57,6 +70,7 @@ export const loadUrlGraph = async ({
   )
 
   const waitAll = async () => {
+    operation.throwIfAborted()
     if (promises.length === 0) {
       return
     }
@@ -66,4 +80,5 @@ export const loadUrlGraph = async ({
     await waitAll()
   }
   await waitAll()
+  promiseMap.clear()
 }

package/src/omega/url_graph/url_graph_report.js

@@ -1,37 +1,36 @@
 import { ANSI } from "@jsenv/log"
 
 import { byteAsFileSize } from "@jsenv/utils/logs/size_log.js"
+import { distributeNumbers } from "@jsenv/utils/logs/number_distribution.js"
 
 export const createUrlGraphSummary = (
   urlGraph,
   { title = "graph summary" } = {},
 ) => {
   const graphReport = createUrlGraphReport(urlGraph)
-  const totalLabel = `Total`
   return `--- ${title} ---  
 ${createRepartitionMessage(graphReport)}
-${ANSI.color(totalLabel, ANSI.GREY)} ${
-    graphReport.total.count
-  } (${byteAsFileSize(graphReport.total.size)})
 --------------------`
 }
 
 const createUrlGraphReport = (urlGraph) => {
   const { urlInfos } = urlGraph
   const countGroups = {
+    sourcemaps: 0,
     html: 0,
     css: 0,
     js: 0,
-    assets: 0,
-    sourcemaps: 0,
+    json: 0,
+    other: 0,
     total: 0,
   }
   const sizeGroups = {
+    sourcemaps: 0,
     html: 0,
     css: 0,
     js: 0,
-    sourcemaps: 0,
-    assets: 0,
+    json: 0,
+    other: 0,
     total: 0,
   }
   Object.keys(urlInfos).forEach((url) => {
@@ -39,8 +38,10 @@ const createUrlGraphReport = (urlGraph) => {
       return
     }
     const urlInfo = urlInfos[url]
-    // ignore inline files, they are already taken into account in the file where they appear
-    if (urlInfo.isInline) {
+    // ignore:
+    // - inline files: they are already taken into account in the file where they appear
+    // - external files: we don't know their content
+    if (urlInfo.isInline || urlInfo.external) {
       return
     }
     // file loaded via import assertion are already inside the graph
@@ -49,9 +50,9 @@ const createUrlGraphReport = (urlGraph) => {
     // and only the js module remain (likely bundled)
     const urlObject = new URL(urlInfo.url)
     if (
-      urlObject.searchParams.has("json_module") ||
-      urlObject.searchParams.has("css_module") ||
-      urlObject.searchParams.has("text_module")
+      urlObject.searchParams.has("as_json_module") ||
+      urlObject.searchParams.has("as_css_module") ||
+      urlObject.searchParams.has("as_text_module")
     ) {
       return
     }
@@ -79,17 +80,67 @@ const createUrlGraphReport = (urlGraph) => {
       sizeGroups.js += urlContentSize
       return
     }
-    countGroups.assets++
-    sizeGroups.assets += urlContentSize
+    if (category === "json") {
+      countGroups.json++
+      sizeGroups.json += urlContentSize
+      return
+    }
+    countGroups.other++
+    sizeGroups.other += urlContentSize
     return
   })
+
+  const sizesToDistribute = {}
+  Object.keys(sizeGroups).forEach((groupName) => {
+    if (groupName !== "sourcemaps" && groupName !== "total") {
+      sizesToDistribute[groupName] = sizeGroups[groupName]
+    }
+  })
+  const ratios = distributeNumbers(sizesToDistribute)
+  const percentageGroups = {}
+  Object.keys(ratios).forEach((groupName) => {
+    percentageGroups[groupName] = parseFloat(ratios[groupName]) * 100
+  })
+
   return {
-    html: { count: countGroups.html, size: sizeGroups.html },
-    css: { count: countGroups.css, size: sizeGroups.css },
-    js: { count: countGroups.js, size: sizeGroups.js },
-    sourcemaps: { count: countGroups.sourcemaps, size: sizeGroups.sourcemaps },
-    assets: { count: countGroups.assets, size: sizeGroups.assets },
-    total: { count: countGroups.total, size: sizeGroups.total },
+    // sourcemaps are special, there size are ignored
+    // so there is no "percentage" associated
+    sourcemaps: {
+      count: countGroups.sourcemaps,
+      size: sizeGroups.sourcemaps,
+      percentage: undefined,
+    },
+
+    html: {
+      count: countGroups.html,
+      size: sizeGroups.html,
+      percentage: percentageGroups.html,
+    },
+    css: {
+      count: countGroups.css,
+      size: sizeGroups.css,
+      percentage: percentageGroups.css,
+    },
+    js: {
+      count: countGroups.js,
+      size: sizeGroups.js,
+      percentage: percentageGroups.js,
+    },
+    json: {
+      count: countGroups.json,
+      size: sizeGroups.json,
+      percentage: percentageGroups.json,
+    },
+    other: {
+      count: countGroups.other,
+      size: sizeGroups.other,
+      percentage: percentageGroups.other,
+    },
+    total: {
+      count: countGroups.total,
+      size: sizeGroups.total,
+      percentage: 100,
+    },
   }
 }
 
@@ -106,32 +157,22 @@ const determineCategory = (urlInfo) => {
   if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
     return "js"
   }
-  return "assets"
+  if (urlInfo.type === "json") {
+    return "json"
+  }
+  return "other"
 }
 
-const createRepartitionMessage = ({ html, css, js, assets }) => {
-  const parts = []
-  if (html.count) {
-    parts.push(
-      `${ANSI.color(`html:`, ANSI.GREY)} ${html.count} (${byteAsFileSize(
-        html.size,
-      )})`,
-    )
-  }
-  if (css.count) {
+const createRepartitionMessage = ({ html, css, js, json, other, total }) => {
+  const addPart = (name, { count, size, percentage }) => {
     parts.push(
-      `${ANSI.color(`css:`, ANSI.GREY)} ${css.count} (${byteAsFileSize(
-        css.size,
-      )})`,
-    )
-  }
-  if (js.count) {
-    parts.push(
-      `${ANSI.color(`js:`, ANSI.GREY)} ${js.count} (${byteAsFileSize(
-        js.size,
-      )})`,
+      `${ANSI.color(`${name}:`, ANSI.GREY)} ${count} (${byteAsFileSize(
+        size,
+      )} / ${percentage} %)`,
     )
   }
+
+  const parts = []
   // if (sourcemaps.count) {
   //   parts.push(
   //     `${ANSI.color(`sourcemaps:`, ANSI.GREY)} ${
@@ -139,13 +180,22 @@ const createRepartitionMessage = ({ html, css, js, assets }) => {
   //     } (${byteAsFileSize(sourcemaps.size)})`,
   //   )
   // }
-  if (assets.count) {
-    parts.push(
-      `${ANSI.color(`assets:`, ANSI.GREY)} ${assets.count} (${byteAsFileSize(
-        assets.size,
-      )})`,
-    )
+  if (html.count) {
+    addPart("html ", html)
+  }
+  if (css.count) {
+    addPart("css  ", css)
+  }
+  if (js.count) {
+    addPart("js   ", js)
+  }
+  if (json.count) {
+    addPart("json ", json)
+  }
+  if (other.count) {
+    addPart("other", other)
   }
+  addPart("total", total)
   return `- ${parts.join(`
 - `)}`
 }

package/src/omega/url_graph/url_info_transformations.js

@@ -1,3 +1,5 @@
+import { bufferToEtag, urlToRelativeUrl } from "@jsenv/filesystem"
+
 import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
 import {
   SOURCEMAP,
@@ -8,7 +10,8 @@ import {
 export const createUrlInfoTransformer = ({
   logger,
   sourcemaps,
-  sourcemapsSources,
+  sourcemapsSourcesContent,
+  sourcemapsRelativeSources,
   urlGraph,
   injectSourcemapPlaceholder,
   foundSourcemap,
@@ -23,7 +26,12 @@ export const createUrlInfoTransformer = ({
       // for inline content (<script> insdide html)
       // chrome won't be able to fetch the file as it does not exists
       // so sourcemap must contain sources
-      urlInfo.isInline || sourcemapsSources
+      sourcemapsSourcesContent ||
+      urlInfo.isInline ||
+      (sourcemap.sources &&
+        sourcemap.sources.some(
+          (source) => !source || !source.startsWith("file:"),
+        ))
     if (sourcemap.sources && sourcemap.sources.length > 1) {
       sourcemap.sources = sourcemap.sources.map(
         (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
@@ -78,9 +86,9 @@ export const createUrlInfoTransformer = ({
       const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
         urlInfo,
         type,
-        line,
-        column,
         specifier,
+        specifierLine: line,
+        specifierColumn: column,
       })
       try {
         await context.cook({
@@ -137,20 +145,29 @@ export const createUrlInfoTransformer = ({
       const sourcemapReference = urlInfo.sourcemapReference
       const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
       sourcemapUrlInfo.contentType = "application/json"
-      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      const sourcemap = urlInfo.sourcemap
+      if (sourcemapsRelativeSources) {
+        sourcemap.sources = sourcemap.sources.map((source) => {
+          const sourceRelative = urlToRelativeUrl(source, urlInfo.url)
+          return sourceRelative
+        })
+      }
+      sourcemapUrlInfo.content = JSON.stringify(sourcemap, null, "  ")
       if (sourcemaps === "inline") {
-        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
-          urlInfo.sourcemap,
-        )
+        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(sourcemap)
       }
       if (sourcemaps === "file" || sourcemaps === "inline") {
         urlInfo.content = SOURCEMAP.writeComment({
           contentType: urlInfo.contentType,
           content: urlInfo.content,
-          specifier: sourcemapReference.generatedSpecifier,
+          specifier:
+            sourcemaps === "file" && sourcemapsRelativeSources
+              ? urlToRelativeUrl(sourcemapReference.url, urlInfo.url)
+              : sourcemapReference.generatedSpecifier,
         })
       }
     }
+    urlInfo.contentEtag = bufferToEtag(Buffer.from(urlInfo.content))
   }
 
   return {

package/src/omega/url_graph.js

@@ -1,10 +1,21 @@
-import { createCallbackList } from "@jsenv/abort"
 import { urlToRelativeUrl } from "@jsenv/filesystem"
 
-export const createUrlGraph = () => {
+export const createUrlGraph = ({
+  clientFileChangeCallbackList,
+  clientFilesPruneCallbackList,
+} = {}) => {
   const urlInfos = {}
   const getUrlInfo = (url) => urlInfos[url]
-  const deleteUrlInfo = (url) => delete urlInfos[url]
+  const deleteUrlInfo = (url) => {
+    const urlInfo = urlInfos[url]
+    if (urlInfo) {
+      delete urlInfos[url]
+      if (urlInfo.sourcemapReference) {
+        deleteUrlInfo(urlInfo.sourcemapReference.url)
+      }
+    }
+  }
+
   const reuseOrCreateUrlInfo = (url) => {
     const existingUrlInfo = urlInfos[url]
     if (existingUrlInfo) return existingUrlInfo
@@ -40,13 +51,22 @@ export const createUrlGraph = () => {
     return visitDependents(urlInfo)
   }
 
-  const prunedCallbackList = createCallbackList()
   const updateReferences = (urlInfo, references) => {
     const dependencyUrls = []
     references.forEach((reference) => {
-      if (!dependencyUrls.includes(reference.url)) {
-        dependencyUrls.push(reference.url)
+      if (reference.isRessourceHint) {
+        // ressource hint are a special kind of reference.
+        // They are a sort of weak reference to an url.
+        // We ignore them so that url referenced only by ressource hints
+        // have url.dependents.size === 0 and can be considered as not used
+        // It means html won't consider url referenced solely
+        // by <link> as dependency and it's fine
+        return
+      }
+      if (dependencyUrls.includes(reference.url)) {
+        return
       }
+      dependencyUrls.push(reference.url)
     })
     pruneDependencies(
       urlInfo,
@@ -82,7 +102,47 @@ export const createUrlGraph = () => {
     if (prunedUrlInfos.length === 0) {
       return
     }
-    prunedCallbackList.notify({ prunedUrlInfos, firstUrlInfo })
+    prunedUrlInfos.forEach((prunedUrlInfo) => {
+      prunedUrlInfo.modifiedTimestamp = Date.now()
+      // should we delete?
+      // delete urlInfos[prunedUrlInfo.url]
+    })
+    if (clientFilesPruneCallbackList) {
+      clientFilesPruneCallbackList.forEach((callback) => {
+        callback({
+          firstUrlInfo,
+          prunedUrlInfos,
+        })
+      })
+    }
+  }
+
+  if (clientFileChangeCallbackList) {
+    const updateModifiedTimestamp = (urlInfo, modifiedTimestamp) => {
+      const seen = []
+      const iterate = (urlInfo) => {
+        if (seen.includes(urlInfo.url)) {
+          return
+        }
+        seen.push(urlInfo.url)
+        urlInfo.modifiedTimestamp = modifiedTimestamp
+        urlInfo.dependents.forEach((dependentUrl) => {
+          const dependentUrlInfo = urlInfos[dependentUrl]
+          const { hotAcceptDependencies = [] } = dependentUrlInfo.data
+          if (!hotAcceptDependencies.includes(urlInfo.url)) {
+            iterate(dependentUrlInfo)
+          }
+        })
+      }
+      iterate(urlInfo)
+    }
+    clientFileChangeCallbackList.push(({ url }) => {
+      const urlInfo = urlInfos[url]
+      if (urlInfo) {
+        updateModifiedTimestamp(urlInfo, Date.now())
+        urlInfo.contentEtag = null
+      }
+    })
   }
 
   return {
@@ -92,8 +152,6 @@ export const createUrlGraph = () => {
     deleteUrlInfo,
     inferReference,
     findDependent,
-
-    prunedCallbackList,
     updateReferences,
 
     toJSON: (rootDirectoryUrl) => {
@@ -114,19 +172,25 @@ export const createUrlGraph = () => {
 
 const createUrlInfo = (url) => {
   return {
+    modifiedTimestamp: 0,
     data: {}, // plugins can put whatever they want here
+    references: [],
+    dependencies: new Set(),
+    dependents: new Set(),
+    type: undefined, // "html", "css", "js_classic", "js_module", "importmap", "json", "webmanifest", ...
+    subtype: undefined, // "worker", "service_worker", "shared_worker" for js, otherwise undefined
+    contentType: "", // "text/html", "text/css", "text/javascript", "application/json", ...
     url,
+    filename: "",
     generatedUrl: null,
     isInline: false,
     inlineUrlSite: null,
-    contentType: "",
-    originalContent: "",
-    content: "",
+    external: false,
+    originalContent: undefined,
+    content: undefined,
+    contentEtag: null,
     sourcemap: null,
     sourcemapReference: null,
-    type: "",
-    references: [],
-    dependencies: new Set(),
-    dependents: new Set(),
+    timing: {},
   }
 }