@jsenv/core 27.0.0-alpha.5 → 27.0.0-alpha.50

package/src/omega/omega_server.js

@@ -8,14 +8,13 @@ import {
 } from "@jsenv/server"
 import { convertFileSystemErrorToResponseProperties } from "@jsenv/server/src/internal/convertFileSystemErrorToResponseProperties.js"
 import { createCallbackListNotifiedOnce } from "@jsenv/abort"
-import { loggerToLogLevel } from "@jsenv/logger"
 
 import { createFileService } from "./server/file_service.js"
 
 export const startOmegaServer = async ({
   signal,
   handleSIGINT,
-  logger,
+  logLevel,
   protocol = "http",
   http2 = protocol === "https",
   privateKey,
@@ -48,7 +47,7 @@ export const startOmegaServer = async ({
     stopOnSIGINT: handleSIGINT,
     stopOnInternalError: false,
     keepProcessAlive,
-    logLevel: loggerToLogLevel(logger),
+    logLevel,
     startLog: false,
 
     protocol,

package/src/omega/server/file_service.js

@@ -39,43 +39,65 @@ export const createFileService = ({
     if (responseFromPlugin) {
       return responseFromPlugin
     }
-    const { runtimeName, runtimeVersion } = parseUserAgentHeader(
-      request.headers["user-agent"],
-    )
-    const runtimeSupport = {
-      [runtimeName]: runtimeVersion,
-    }
-    const reference = kitchen.createReference({
+    const [reference, urlInfo] = kitchen.prepareEntryPoint({
       parentUrl: inferParentFromRequest(request, rootDirectoryUrl),
       type: "entry_point",
       specifier: request.ressource,
     })
-    const requestedUrlInfo = kitchen.resolveReference(reference)
+    const ifNoneMatch = request.headers["if-none-match"]
+    if (ifNoneMatch && urlInfo.contentEtag === ifNoneMatch) {
+      return {
+        status: 304,
+        headers: {
+          "cache-control": `private,max-age=0,must-revalidate`,
+        },
+      }
+    }
     const referenceFromGraph = urlGraph.inferReference(
       reference.url,
       reference.parentUrl,
     )
     try {
+      // urlInfo objects are reused, they must be "reset" before cooking then again
+      if (!urlInfo.isInline && !urlInfo.type === "sourcemap") {
+        urlGraph.resetUrlInfo(urlInfo)
+      }
+      const { runtimeName, runtimeVersion } = parseUserAgentHeader(
+        request.headers["user-agent"],
+      )
       await kitchen.cook({
         reference: referenceFromGraph || reference,
-        urlInfo: requestedUrlInfo,
+        urlInfo,
         outDirectoryUrl: `${rootDirectoryUrl}.jsenv/${scenario}/${runtimeName}@${runtimeVersion}/`,
-        runtimeSupport,
+        clientRuntimeCompat: {
+          [runtimeName]: runtimeVersion,
+        },
       })
-      const { response, contentType, content } = requestedUrlInfo
+      let { response, contentType, content, contentEtag } = urlInfo
       if (response) {
         return response
       }
-      return {
+      response = {
         url: reference.url,
         status: 200,
         headers: {
           "content-type": contentType,
           "content-length": Buffer.byteLength(content),
           "cache-control": `private,max-age=0,must-revalidate`,
+          "eTag": contentEtag,
         },
         body: content,
+        timing: urlInfo.timing,
       }
+      kitchen.pluginController.callHooks(
+        "augmentResponse",
+        { reference, urlInfo },
+        {},
+        (returnValue) => {
+          response = composeTwoResponses(response, returnValue)
+        },
+      )
+      return response
     } catch (e) {
       const code = e.code
       if (code === "PARSE_ERROR") {
@@ -86,11 +108,11 @@ export const createFileService = ({
           statusText: e.reason,
           statusMessage: e.message,
           headers: {
-            "content-type": requestedUrlInfo.contentType,
-            "content-length": Buffer.byteLength(requestedUrlInfo.content),
+            "content-type": urlInfo.contentType,
+            "content-length": Buffer.byteLength(urlInfo.content),
             "cache-control": "no-store",
           },
-          body: requestedUrlInfo.content,
+          body: urlInfo.content,
         }
       }
       if (code === "EISDIR") {
@@ -127,16 +149,7 @@ export const createFileService = ({
   }
   return async (request) => {
     let response = await getResponse(request)
-    if (response.url) {
-      kitchen.pluginController.callHooks(
-        "augmentResponse",
-        response,
-        {},
-        (returnValue) => {
-          response = composeTwoResponses(response, returnValue)
-        },
-      )
-    }
+
     return response
   }
 }

package/src/omega/server/user_agent.js

@@ -1,8 +1,10 @@
 import { createRequire } from "node:module"
 
+import { memoizeByFirstArgument } from "@jsenv/utils/memoize/memoize_by_first_argument.js"
+
 const require = createRequire(import.meta.url)
 
-export const parseUserAgentHeader = (userAgent) => {
+export const parseUserAgentHeader = memoizeByFirstArgument((userAgent) => {
   if (userAgent.includes("node-fetch/")) {
     // it's not really node and conceptually we can't assume the node version
     // but good enough for now
@@ -19,4 +21,4 @@ export const parseUserAgentHeader = (userAgent) => {
     runtimeVersion:
       family === "Other" ? "unknown" : `${major}.${minor}${patch}`,
   }
-}
+})

package/src/omega/url_graph/url_graph_load.js

@@ -5,23 +5,24 @@ export const loadUrlGraph = async ({
   kitchen,
   startLoading,
   outDirectoryUrl,
-  runtimeSupport,
+  clientRuntimeCompat,
 }) => {
   if (outDirectoryUrl) {
     await ensureEmptyDirectory(outDirectoryUrl)
   }
   const promises = []
+  const promiseMap = new Map()
   const cook = ({ urlInfo, ...rest }) => {
-    const promiseFromData = urlInfo.data.promise
+    const promiseFromData = promiseMap.get(urlInfo)
     if (promiseFromData) return promiseFromData
     const promise = _cook({
       urlInfo,
       outDirectoryUrl,
-      runtimeSupport,
+      clientRuntimeCompat,
       ...rest,
     })
     promises.push(promise)
-    urlInfo.data.promise = promise
+    promiseMap.set(urlInfo, promise)
     return promise
   }
   const _cook = async ({ urlInfo, ...rest }) => {
@@ -32,26 +33,31 @@ export const loadUrlGraph = async ({
     })
     const { references } = urlInfo
     references.forEach((reference) => {
+      // we use reference.generatedUrl to mimic what a browser would do:
+      // do a fetch to the specifier as found in the file
+      const referencedUrlInfo = urlGraph.reuseOrCreateUrlInfo(
+        reference.generatedUrl,
+      )
       cook({
         reference,
-        urlInfo: urlGraph.getUrlInfo(reference.url),
+        urlInfo: referencedUrlInfo,
       })
     })
   }
   startLoading(
     ({ trace, parentUrl = kitchen.rootDirectoryUrl, type, specifier }) => {
-      const entryReference = kitchen.createReference({
+      const [entryReference, entryUrlInfo] = kitchen.prepareEntryPoint({
         trace,
         parentUrl,
         type,
         specifier,
       })
-      const entryUrlInfo = kitchen.resolveReference(entryReference)
       entryUrlInfo.data.isEntryPoint = true
       cook({
         reference: entryReference,
         urlInfo: entryUrlInfo,
       })
+      return [entryReference, entryUrlInfo]
     },
   )
 
@@ -65,4 +71,5 @@ export const loadUrlGraph = async ({
     await waitAll()
   }
   await waitAll()
+  promiseMap.clear()
 }

package/src/omega/url_graph/url_graph_report.js

@@ -22,7 +22,7 @@ const createUrlGraphReport = (urlGraph) => {
     html: 0,
     css: 0,
     js: 0,
-    assets: 0,
+    other: 0,
     sourcemaps: 0,
     total: 0,
   }
@@ -31,7 +31,7 @@ const createUrlGraphReport = (urlGraph) => {
     css: 0,
     js: 0,
     sourcemaps: 0,
-    assets: 0,
+    other: 0,
     total: 0,
   }
   Object.keys(urlInfos).forEach((url) => {
@@ -39,8 +39,10 @@ const createUrlGraphReport = (urlGraph) => {
       return
     }
     const urlInfo = urlInfos[url]
-    // ignore inline files, they are already taken into account in the file where they appear
-    if (urlInfo.isInline) {
+    // ignore:
+    // - inline files: they are already taken into account in the file where they appear
+    // - external files: we don't know their content
+    if (urlInfo.isInline || urlInfo.external) {
       return
     }
     // file loaded via import assertion are already inside the graph
@@ -49,9 +51,9 @@ const createUrlGraphReport = (urlGraph) => {
     // and only the js module remain (likely bundled)
     const urlObject = new URL(urlInfo.url)
     if (
-      urlObject.searchParams.has("json_module") ||
-      urlObject.searchParams.has("css_module") ||
-      urlObject.searchParams.has("text_module")
+      urlObject.searchParams.has("as_json_module") ||
+      urlObject.searchParams.has("as_css_module") ||
+      urlObject.searchParams.has("as_text_module")
     ) {
       return
     }
@@ -79,8 +81,8 @@ const createUrlGraphReport = (urlGraph) => {
       sizeGroups.js += urlContentSize
       return
     }
-    countGroups.assets++
-    sizeGroups.assets += urlContentSize
+    countGroups.other++
+    sizeGroups.other += urlContentSize
     return
   })
   return {
@@ -88,7 +90,7 @@ const createUrlGraphReport = (urlGraph) => {
     css: { count: countGroups.css, size: sizeGroups.css },
     js: { count: countGroups.js, size: sizeGroups.js },
     sourcemaps: { count: countGroups.sourcemaps, size: sizeGroups.sourcemaps },
-    assets: { count: countGroups.assets, size: sizeGroups.assets },
+    other: { count: countGroups.other, size: sizeGroups.other },
     total: { count: countGroups.total, size: sizeGroups.total },
   }
 }
@@ -106,10 +108,10 @@ const determineCategory = (urlInfo) => {
   if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
     return "js"
   }
-  return "assets"
+  return "other"
 }
 
-const createRepartitionMessage = ({ html, css, js, assets }) => {
+const createRepartitionMessage = ({ html, css, js, other }) => {
   const parts = []
   if (html.count) {
     parts.push(
@@ -139,10 +141,10 @@ const createRepartitionMessage = ({ html, css, js, assets }) => {
   //     } (${byteAsFileSize(sourcemaps.size)})`,
   //   )
   // }
-  if (assets.count) {
+  if (other.count) {
     parts.push(
-      `${ANSI.color(`assets:`, ANSI.GREY)} ${assets.count} (${byteAsFileSize(
-        assets.size,
+      `${ANSI.color(`other:`, ANSI.GREY)} ${other.count} (${byteAsFileSize(
+        other.size,
       )})`,
     )
   }

package/src/omega/url_graph/url_info_transformations.js

@@ -1,3 +1,5 @@
+import { bufferToEtag, urlToRelativeUrl } from "@jsenv/filesystem"
+
 import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
 import {
   SOURCEMAP,
@@ -8,7 +10,8 @@ import {
 export const createUrlInfoTransformer = ({
   logger,
   sourcemaps,
-  sourcemapsSources,
+  sourcemapsSourcesContent,
+  sourcemapsRelativeSources,
   urlGraph,
   injectSourcemapPlaceholder,
   foundSourcemap,
@@ -23,7 +26,12 @@ export const createUrlInfoTransformer = ({
       // for inline content (<script> insdide html)
       // chrome won't be able to fetch the file as it does not exists
       // so sourcemap must contain sources
-      urlInfo.isInline || sourcemapsSources
+      sourcemapsSourcesContent ||
+      urlInfo.isInline ||
+      (sourcemap.sources &&
+        sourcemap.sources.some(
+          (source) => !source || !source.startsWith("file:"),
+        ))
     if (sourcemap.sources && sourcemap.sources.length > 1) {
       sourcemap.sources = sourcemap.sources.map(
         (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
@@ -78,9 +86,9 @@ export const createUrlInfoTransformer = ({
       const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
         urlInfo,
         type,
-        line,
-        column,
         specifier,
+        specifierLine: line,
+        specifierColumn: column,
       })
       try {
         await context.cook({
@@ -137,20 +145,29 @@ export const createUrlInfoTransformer = ({
       const sourcemapReference = urlInfo.sourcemapReference
       const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
       sourcemapUrlInfo.contentType = "application/json"
-      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      const sourcemap = urlInfo.sourcemap
+      if (sourcemapsRelativeSources) {
+        sourcemap.sources = sourcemap.sources.map((source) => {
+          const sourceRelative = urlToRelativeUrl(source, urlInfo.url)
+          return sourceRelative
+        })
+      }
+      sourcemapUrlInfo.content = JSON.stringify(sourcemap, null, "  ")
       if (sourcemaps === "inline") {
-        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
-          urlInfo.sourcemap,
-        )
+        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(sourcemap)
       }
       if (sourcemaps === "file" || sourcemaps === "inline") {
         urlInfo.content = SOURCEMAP.writeComment({
           contentType: urlInfo.contentType,
           content: urlInfo.content,
-          specifier: sourcemapReference.generatedSpecifier,
+          specifier:
+            sourcemaps === "file" && sourcemapsRelativeSources
+              ? urlToRelativeUrl(sourcemapReference.url, urlInfo.url)
+              : sourcemapReference.generatedSpecifier,
         })
       }
     }
+    urlInfo.contentEtag = bufferToEtag(Buffer.from(urlInfo.content))
   }
 
   return {

package/src/omega/url_graph.js

@@ -1,10 +1,31 @@
-import { createCallbackList } from "@jsenv/abort"
 import { urlToRelativeUrl } from "@jsenv/filesystem"
 
-export const createUrlGraph = () => {
+export const createUrlGraph = ({
+  clientFileChangeCallbackList,
+  clientFilesPruneCallbackList,
+} = {}) => {
   const urlInfos = {}
   const getUrlInfo = (url) => urlInfos[url]
-  const deleteUrlInfo = (url) => delete urlInfos[url]
+  const deleteUrlInfo = (url) => {
+    const urlInfo = urlInfos[url]
+    if (urlInfo) {
+      delete urlInfos[url]
+      if (urlInfo.sourcemapReference) {
+        deleteUrlInfo(urlInfo.sourcemapReference.url)
+      }
+    }
+  }
+  const resetUrlInfo = (urlInfo) => {
+    urlInfo.sourcemap = null
+    urlInfo.sourcemapReference = null
+    urlInfo.content = null
+    urlInfo.originalContent = null
+    urlInfo.type = null
+    urlInfo.subtype = null
+    urlInfo.data = {}
+    urlInfo.timing = {}
+  }
+
   const reuseOrCreateUrlInfo = (url) => {
     const existingUrlInfo = urlInfos[url]
     if (existingUrlInfo) return existingUrlInfo
@@ -40,13 +61,22 @@ export const createUrlGraph = () => {
     return visitDependents(urlInfo)
   }
 
-  const prunedCallbackList = createCallbackList()
   const updateReferences = (urlInfo, references) => {
     const dependencyUrls = []
     references.forEach((reference) => {
-      if (!dependencyUrls.includes(reference.url)) {
-        dependencyUrls.push(reference.url)
+      if (reference.isRessourceHint) {
+        // ressource hint are a special kind of reference.
+        // They are a sort of weak reference to an url.
+        // We ignore them so that url referenced only by ressource hints
+        // have url.dependents.size === 0 and can be considered as not used
+        // It means html won't consider url referenced solely
+        // by <link> as dependency and it's fine
+        return
+      }
+      if (dependencyUrls.includes(reference.url)) {
+        return
       }
+      dependencyUrls.push(reference.url)
     })
     pruneDependencies(
       urlInfo,
@@ -82,7 +112,47 @@ export const createUrlGraph = () => {
     if (prunedUrlInfos.length === 0) {
       return
     }
-    prunedCallbackList.notify({ prunedUrlInfos, firstUrlInfo })
+    prunedUrlInfos.forEach((prunedUrlInfo) => {
+      prunedUrlInfo.modifiedTimestamp = Date.now()
+      // should we delete?
+      // delete urlInfos[prunedUrlInfo.url]
+    })
+    if (clientFilesPruneCallbackList) {
+      clientFilesPruneCallbackList.forEach((callback) => {
+        callback({
+          firstUrlInfo,
+          prunedUrlInfos,
+        })
+      })
+    }
+  }
+
+  if (clientFileChangeCallbackList) {
+    const updateModifiedTimestamp = (urlInfo, modifiedTimestamp) => {
+      const seen = []
+      const iterate = (urlInfo) => {
+        if (seen.includes(urlInfo.url)) {
+          return
+        }
+        seen.push(urlInfo.url)
+        urlInfo.modifiedTimestamp = modifiedTimestamp
+        urlInfo.dependents.forEach((dependentUrl) => {
+          const dependentUrlInfo = urlInfos[dependentUrl]
+          const { hotAcceptDependencies = [] } = dependentUrlInfo.data
+          if (!hotAcceptDependencies.includes(urlInfo.url)) {
+            iterate(dependentUrlInfo)
+          }
+        })
+      }
+      iterate(urlInfo)
+    }
+    clientFileChangeCallbackList.push(({ url }) => {
+      const urlInfo = urlInfos[url]
+      if (urlInfo) {
+        updateModifiedTimestamp(urlInfo, Date.now())
+        urlInfo.contentEtag = null
+      }
+    })
   }
 
   return {
@@ -90,10 +160,9 @@ export const createUrlGraph = () => {
     reuseOrCreateUrlInfo,
     getUrlInfo,
     deleteUrlInfo,
+    resetUrlInfo,
     inferReference,
     findDependent,
-
-    prunedCallbackList,
     updateReferences,
 
     toJSON: (rootDirectoryUrl) => {
@@ -114,19 +183,25 @@ export const createUrlGraph = () => {
 
 const createUrlInfo = (url) => {
   return {
+    modifiedTimestamp: 0,
     data: {}, // plugins can put whatever they want here
+    references: [],
+    dependencies: new Set(),
+    dependents: new Set(),
+    type: undefined, // "html", "css", "js_classic", "js_module", "importmap", "json", "webmanifest", ...
+    subtype: undefined, // "worker", "service_worker", "shared_worker" for js, otherwise undefined
+    contentType: "", // "text/html", "text/css", "text/javascript", "application/json", ...
     url,
+    filename: "",
     generatedUrl: null,
     isInline: false,
     inlineUrlSite: null,
-    contentType: "",
-    originalContent: "",
-    content: "",
+    external: false,
+    originalContent: undefined,
+    content: undefined,
+    contentEtag: null,
     sourcemap: null,
     sourcemapReference: null,
-    type: "",
-    references: [],
-    dependencies: new Set(),
-    dependents: new Set(),
+    timing: {},
   }
 }

package/src/omega/web_workers.js

@@ -0,0 +1,42 @@
+// the following apis are creating js entry points:
+// - new Worker()
+// - new SharedWorker()
+// - navigator.serviceWorker.register()
+export const isWebWorkerEntryPointReference = (reference) => {
+  if (reference.subtype === "new_url_first_arg") {
+    return ["worker", "service_worker", "shared_worker"].includes(
+      reference.expectedSubtype,
+    )
+  }
+  return [
+    "new_worker_first_arg",
+    "new_shared_worker_first_arg",
+    "service_worker_register_first_arg",
+  ].includes(reference.subtype)
+}
+
+export const isWebWorkerUrlInfo = (urlInfo) => {
+  return (
+    urlInfo.subtype === "worker" ||
+    urlInfo.subtype === "service_worker" ||
+    urlInfo.subtype === "shared_worker"
+  )
+}
+
+// export const isEntryPoint = (urlInfo, urlGraph) => {
+//   if (urlInfo.data.isEntryPoint) {
+//     return true
+//   }
+//   if (isWebWorker(urlInfo)) {
+//     // - new Worker("a.js") -> "a.js" is an entry point
+//     // - self.importScripts("b.js") -> "b.js" is not an entry point
+//     // So the following logic applies to infer if the file is a web worker entry point
+//     // "When a non-webworker file references a worker file, the worker file is an entry point"
+//     const dependents = Array.from(urlInfo.dependents)
+//     return dependents.some((dependentUrl) => {
+//       const dependentUrlInfo = urlGraph.getUrlInfo(dependentUrl)
+//       return !isWebWorker(dependentUrlInfo)
+//     })
+//   }
+//   return false
+// }

package/src/{dev/plugins/autoreload → plugins/autoreload/dev_sse}/client/event_source_client.js

@@ -1,4 +1,5 @@
-import { createEventSourceConnection } from "./event_source_connection.js"
+import { createEventSourceConnection } from "@jsenv/utils/event_source/event_source.js"
+import { urlHotMetas } from "../../../import_meta_hot/client/import_meta_hot.js"
 import {
   isAutoreloadEnabled,
   setAutoreloadPreference,
@@ -9,7 +10,6 @@ import {
   reloadDOMNodesUsingUrl,
   reloadJsImport,
 } from "./reload.js"
-import { urlHotMetas } from "./import_meta_hot.js"
 
 const reloadMessages = []
 const reloadMessagesSignal = { onchange: () => {} }

package/src/plugins/autoreload/dev_sse/jsenv_plugin_dev_sse_client.js

@@ -0,0 +1,46 @@
+import { urlIsInsideOf } from "@jsenv/filesystem"
+
+import {
+  parseHtmlString,
+  stringifyHtmlAst,
+  injectScriptAsEarlyAsPossible,
+  createHtmlNode,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import { jsenvRootDirectoryUrl } from "@jsenv/core/src/jsenv_root_directory_url.js"
+
+export const jsenvPluginDevSSEClient = ({ rootDirectoryUrl }) => {
+  const preferSourceFiles =
+    rootDirectoryUrl === jsenvRootDirectoryUrl ||
+    urlIsInsideOf(rootDirectoryUrl, jsenvRootDirectoryUrl)
+  const eventSourceClientFileUrl = preferSourceFiles
+    ? new URL("./client/event_source_client.js", import.meta.url).href
+    : new URL("./dist/event_source_client.js", jsenvRootDirectoryUrl).href
+
+  return {
+    name: "jsenv:dev_sse_client",
+    appliesDuring: { dev: true },
+    transformUrlContent: {
+      html: (htmlUrlInfo, context) => {
+        const htmlAst = parseHtmlString(htmlUrlInfo.content)
+        const [eventSourceClientReference] = context.referenceUtils.inject({
+          type: "script_src",
+          expectedType: "js_module",
+          specifier: eventSourceClientFileUrl,
+        })
+        injectScriptAsEarlyAsPossible(
+          htmlAst,
+          createHtmlNode({
+            "tagName": "script",
+            "type": "module",
+            "src": eventSourceClientReference.generatedSpecifier,
+            "injected-by": "jsenv:dev_sse_client",
+          }),
+        )
+        const htmlModified = stringifyHtmlAst(htmlAst)
+        return {
+          content: htmlModified,
+        }
+      },
+    },
+  }
+}