@jsenv/core 27.0.0-alpha.9 → 27.0.0-alpha.92

package/src/omega/url_graph/url_info_transformations.js

@@ -1,14 +1,18 @@
-import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+import { pathToFileURL } from "node:url"
+import { bufferToEtag } from "@jsenv/filesystem"
+import { urlToRelativeUrl, isFileSystemPath } from "@jsenv/urls"
 import {
+  composeTwoSourcemaps,
   SOURCEMAP,
-  sourcemapToBase64Url,
-  generateSourcemapUrl,
-} from "@jsenv/utils/sourcemap/sourcemap_utils.js"
+  generateSourcemapFileUrl,
+  generateSourcemapDataUrl,
+} from "@jsenv/sourcemap"
 
 export const createUrlInfoTransformer = ({
   logger,
   sourcemaps,
-  sourcemapsSources,
+  sourcemapsSourcesContent,
+  sourcemapsRelativeSources,
   urlGraph,
   injectSourcemapPlaceholder,
   foundSourcemap,
@@ -19,21 +23,33 @@ export const createUrlInfoTransformer = ({
     sourcemaps === "programmatic"
 
   const normalizeSourcemap = (urlInfo, sourcemap) => {
+    let { sources } = sourcemap
+    if (sources) {
+      sources = sources.map((source) => {
+        if (source && isFileSystemPath(source)) {
+          return String(pathToFileURL(source))
+        }
+        return source
+      })
+    }
     const wantSourcesContent =
       // for inline content (<script> insdide html)
       // chrome won't be able to fetch the file as it does not exists
       // so sourcemap must contain sources
-      urlInfo.isInline || sourcemapsSources
-    if (sourcemap.sources && sourcemap.sources.length > 1) {
-      sourcemap.sources = sourcemap.sources.map(
-        (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
+      sourcemapsSourcesContent ||
+      urlInfo.isInline ||
+      (sources &&
+        sources.some((source) => !source || !source.startsWith("file:")))
+    if (sources && sources.length > 1) {
+      sourcemap.sources = sources.map(
+        (source) => new URL(source, urlInfo.originalUrl).href,
       )
       if (!wantSourcesContent) {
         sourcemap.sourcesContent = undefined
       }
       return sourcemap
     }
-    sourcemap.sources = [urlInfo.data.rawUrl || urlInfo.url]
+    sourcemap.sources = [urlInfo.originalUrl]
     sourcemap.sourcesContent = [urlInfo.originalContent]
     if (!wantSourcesContent) {
       sourcemap.sourcesContent = undefined
@@ -56,7 +72,9 @@ export const createUrlInfoTransformer = ({
     // when jsenv is done cooking the file
     //   during build it's urlInfo.url to be inside the build
     //   but otherwise it's generatedUrl to be inside .jsenv/ directory
-    urlInfo.sourcemapGeneratedUrl = generateSourcemapUrl(urlInfo.generatedUrl)
+    urlInfo.sourcemapGeneratedUrl = generateSourcemapFileUrl(
+      urlInfo.generatedUrl,
+    )
     const [sourcemapReference, sourcemapUrlInfo] = injectSourcemapPlaceholder({
       urlInfo,
       specifier: urlInfo.sourcemapGeneratedUrl,
@@ -66,6 +84,7 @@ export const createUrlInfoTransformer = ({
 
     // already loaded during "load" hook (happens during build)
     if (urlInfo.sourcemap) {
+      urlInfo.sourcemap = normalizeSourcemap(urlInfo, urlInfo.sourcemap)
       return
     }
     // check for existing sourcemap for this content
@@ -78,17 +97,15 @@ export const createUrlInfoTransformer = ({
       const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
         urlInfo,
         type,
-        line,
-        column,
         specifier,
+        specifierLine: line,
+        specifierColumn: column,
       })
       try {
-        await context.cook({
-          reference: sourcemapReference,
-          urlInfo: sourcemapUrlInfo,
-        })
-        const sourcemap = JSON.parse(sourcemapUrlInfo.content)
-        urlInfo.sourcemap = normalizeSourcemap(urlInfo, sourcemap)
+        await context.cook(sourcemapUrlInfo, { reference: sourcemapReference })
+        const sourcemapRaw = JSON.parse(sourcemapUrlInfo.content)
+        const sourcemap = normalizeSourcemap(urlInfo, sourcemapRaw)
+        urlInfo.sourcemap = sourcemap
       } catch (e) {
         logger.error(`Error while handling existing sourcemap: ${e.message}`)
         return
@@ -137,20 +154,33 @@ export const createUrlInfoTransformer = ({
       const sourcemapReference = urlInfo.sourcemapReference
       const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
       sourcemapUrlInfo.contentType = "application/json"
-      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      const sourcemap = urlInfo.sourcemap
+      if (sourcemapsRelativeSources) {
+        sourcemap.sources = sourcemap.sources.map((source) => {
+          const sourceRelative = urlToRelativeUrl(source, urlInfo.url)
+          return sourceRelative || "."
+        })
+      }
+      sourcemapUrlInfo.content = JSON.stringify(sourcemap, null, "  ")
       if (sourcemaps === "inline") {
-        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
-          urlInfo.sourcemap,
-        )
+        sourcemapReference.generatedSpecifier =
+          generateSourcemapDataUrl(sourcemap)
       }
       if (sourcemaps === "file" || sourcemaps === "inline") {
         urlInfo.content = SOURCEMAP.writeComment({
           contentType: urlInfo.contentType,
           content: urlInfo.content,
-          specifier: sourcemapReference.generatedSpecifier,
+          specifier:
+            sourcemaps === "file" && sourcemapsRelativeSources
+              ? urlToRelativeUrl(sourcemapReference.url, urlInfo.url)
+              : sourcemapReference.generatedSpecifier,
         })
       }
+    } else if (urlInfo.sourcemapReference) {
+      // in the end we don't use the sourcemap placeholder
+      urlGraph.deleteUrlInfo(urlInfo.sourcemapReference.url)
     }
+    urlInfo.contentEtag = bufferToEtag(Buffer.from(urlInfo.content))
   }
 
   return {

package/src/omega/url_graph.js

@@ -1,10 +1,22 @@
-import { createCallbackList } from "@jsenv/abort"
-import { urlToRelativeUrl } from "@jsenv/filesystem"
+import { urlToRelativeUrl } from "@jsenv/urls"
+import { urlSpecifierEncoding } from "./url_specifier_encoding.js"
 
-export const createUrlGraph = () => {
+export const createUrlGraph = ({
+  clientFileChangeCallbackList,
+  clientFilesPruneCallbackList,
+} = {}) => {
   const urlInfos = {}
   const getUrlInfo = (url) => urlInfos[url]
-  const deleteUrlInfo = (url) => delete urlInfos[url]
+  const deleteUrlInfo = (url) => {
+    const urlInfo = urlInfos[url]
+    if (urlInfo) {
+      delete urlInfos[url]
+      if (urlInfo.sourcemapReference) {
+        deleteUrlInfo(urlInfo.sourcemapReference.url)
+      }
+    }
+  }
+
   const reuseOrCreateUrlInfo = (url) => {
     const existingUrlInfo = urlInfos[url]
     if (existingUrlInfo) return existingUrlInfo
@@ -12,13 +24,15 @@ export const createUrlGraph = () => {
     urlInfos[url] = urlInfo
     return urlInfo
   }
-  const inferReference = (url, parentUrl) => {
+  const inferReference = (specifier, parentUrl) => {
     const parentUrlInfo = urlInfos[parentUrl]
     if (!parentUrlInfo) {
       return null
     }
     const firstReferenceOnThatUrl = parentUrlInfo.references.find(
-      (reference) => reference.url === url,
+      (reference) => {
+        return urlSpecifierEncoding.decode(reference) === specifier
+      },
     )
     return firstReferenceOnThatUrl
   }
@@ -40,13 +54,22 @@ export const createUrlGraph = () => {
     return visitDependents(urlInfo)
   }
 
-  const prunedCallbackList = createCallbackList()
   const updateReferences = (urlInfo, references) => {
     const dependencyUrls = []
     references.forEach((reference) => {
-      if (!dependencyUrls.includes(reference.url)) {
-        dependencyUrls.push(reference.url)
+      if (reference.isRessourceHint) {
+        // ressource hint are a special kind of reference.
+        // They are a sort of weak reference to an url.
+        // We ignore them so that url referenced only by ressource hints
+        // have url.dependents.size === 0 and can be considered as not used
+        // It means html won't consider url referenced solely
+        // by <link> as dependency and it's fine
+        return
+      }
+      if (dependencyUrls.includes(reference.url)) {
+        return
       }
+      dependencyUrls.push(reference.url)
     })
     pruneDependencies(
       urlInfo,
@@ -82,7 +105,47 @@ export const createUrlGraph = () => {
     if (prunedUrlInfos.length === 0) {
       return
     }
-    prunedCallbackList.notify({ prunedUrlInfos, firstUrlInfo })
+    prunedUrlInfos.forEach((prunedUrlInfo) => {
+      prunedUrlInfo.modifiedTimestamp = Date.now()
+      // should we delete?
+      // delete urlInfos[prunedUrlInfo.url]
+    })
+    if (clientFilesPruneCallbackList) {
+      clientFilesPruneCallbackList.forEach((callback) => {
+        callback({
+          firstUrlInfo,
+          prunedUrlInfos,
+        })
+      })
+    }
+  }
+
+  if (clientFileChangeCallbackList) {
+    const updateModifiedTimestamp = (urlInfo, modifiedTimestamp) => {
+      const seen = []
+      const iterate = (urlInfo) => {
+        if (seen.includes(urlInfo.url)) {
+          return
+        }
+        seen.push(urlInfo.url)
+        urlInfo.modifiedTimestamp = modifiedTimestamp
+        urlInfo.dependents.forEach((dependentUrl) => {
+          const dependentUrlInfo = urlInfos[dependentUrl]
+          const { hotAcceptDependencies = [] } = dependentUrlInfo.data
+          if (!hotAcceptDependencies.includes(urlInfo.url)) {
+            iterate(dependentUrlInfo)
+          }
+        })
+      }
+      iterate(urlInfo)
+    }
+    clientFileChangeCallbackList.push(({ url }) => {
+      const urlInfo = urlInfos[url]
+      if (urlInfo) {
+        updateModifiedTimestamp(urlInfo, Date.now())
+        urlInfo.contentEtag = null
+      }
+    })
   }
 
   return {
@@ -92,8 +155,6 @@ export const createUrlGraph = () => {
     deleteUrlInfo,
     inferReference,
     findDependent,
-
-    prunedCallbackList,
     updateReferences,
 
     toJSON: (rootDirectoryUrl) => {
@@ -114,19 +175,27 @@ export const createUrlGraph = () => {
 
 const createUrlInfo = (url) => {
   return {
+    modifiedTimestamp: 0,
     data: {}, // plugins can put whatever they want here
+    references: [],
+    dependencies: new Set(),
+    dependents: new Set(),
+    type: undefined, // "html", "css", "js_classic", "js_module", "importmap", "json", "webmanifest", ...
+    subtype: undefined, // "worker", "service_worker", "shared_worker" for js, otherwise undefined
+    contentType: "", // "text/html", "text/css", "text/javascript", "application/json", ...
     url,
+    originalUrl: undefined,
     generatedUrl: null,
+    filename: "",
     isInline: false,
     inlineUrlSite: null,
-    contentType: "",
-    originalContent: "",
-    content: "",
+    shouldHandle: undefined,
+    originalContent: undefined,
+    content: undefined,
+    contentEtag: null,
     sourcemap: null,
     sourcemapReference: null,
-    type: "",
-    references: [],
-    dependencies: new Set(),
-    dependents: new Set(),
+    timing: {},
+    headers: {},
   }
 }

package/src/omega/url_specifier_encoding.js

@@ -0,0 +1,59 @@
+export const urlSpecifierEncoding = {
+  encode: (reference) => {
+    const { generatedSpecifier } = reference
+    if (generatedSpecifier.then) {
+      return generatedSpecifier.then((value) => {
+        reference.generatedSpecifier = value
+        return urlSpecifierEncoding.encode(reference)
+      })
+    }
+    // allow plugin to return a function to bypas default formatting
+    // (which is to use JSON.stringify when url is referenced inside js)
+    if (typeof generatedSpecifier === "function") {
+      return generatedSpecifier()
+    }
+    const formatter = formatters[reference.type]
+    const value = formatter
+      ? formatter.encode(generatedSpecifier)
+      : generatedSpecifier
+    if (reference.escape) {
+      return reference.escape(value)
+    }
+    return value
+  },
+  decode: (reference) => {
+    const formatter = formatters[reference.type]
+    return formatter
+      ? formatter.decode(reference.generatedSpecifier)
+      : reference.generatedSpecifier
+  },
+}
+const formatters = {
+  "js_import_export": { encode: JSON.stringify, decode: JSON.parse },
+  "js_url_specifier": { encode: JSON.stringify, decode: JSON.parse },
+  "css_@import": { encode: JSON.stringify, code: JSON.stringify },
+  // https://github.com/webpack-contrib/css-loader/pull/627/files
+  "css_url": {
+    encode: (url) => {
+      // If url is already wrapped in quotes, remove them
+      url = formatters.css_url.decode(url)
+      // Should url be wrapped?
+      // See https://drafts.csswg.org/css-values-3/#urls
+      if (/["'() \t\n]/.test(url)) {
+        return `"${url.replace(/"/g, '\\"').replace(/\n/g, "\\n")}"`
+      }
+      return url
+    },
+    decode: (url) => {
+      const firstChar = url[0]
+      const lastChar = url[url.length - 1]
+      if (firstChar === `"` && lastChar === `"`) {
+        return url.slice(1, -1)
+      }
+      if (firstChar === `'` && lastChar === `'`) {
+        return url.slice(1, -1)
+      }
+      return url
+    },
+  },
+}

package/src/omega/web_workers.js

@@ -0,0 +1,42 @@
+// the following apis are creating js entry points:
+// - new Worker()
+// - new SharedWorker()
+// - navigator.serviceWorker.register()
+export const isWebWorkerEntryPointReference = (reference) => {
+  if (reference.subtype === "new_url_first_arg") {
+    return ["worker", "service_worker", "shared_worker"].includes(
+      reference.expectedSubtype,
+    )
+  }
+  return [
+    "new_worker_first_arg",
+    "new_shared_worker_first_arg",
+    "service_worker_register_first_arg",
+  ].includes(reference.subtype)
+}
+
+export const isWebWorkerUrlInfo = (urlInfo) => {
+  return (
+    urlInfo.subtype === "worker" ||
+    urlInfo.subtype === "service_worker" ||
+    urlInfo.subtype === "shared_worker"
+  )
+}
+
+// export const isEntryPoint = (urlInfo, urlGraph) => {
+//   if (urlInfo.data.isEntryPoint) {
+//     return true
+//   }
+//   if (isWebWorker(urlInfo)) {
+//     // - new Worker("a.js") -> "a.js" is an entry point
+//     // - self.importScripts("b.js") -> "b.js" is not an entry point
+//     // So the following logic applies to infer if the file is a web worker entry point
+//     // "When a non-webworker file references a worker file, the worker file is an entry point"
+//     const dependents = Array.from(urlInfo.dependents)
+//     return dependents.some((dependentUrl) => {
+//       const dependentUrlInfo = urlGraph.getUrlInfo(dependentUrl)
+//       return !isWebWorker(dependentUrlInfo)
+//     })
+//   }
+//   return false
+// }

package/src/{dev/plugins/autoreload → plugins/autoreload/dev_sse}/client/event_source_client.js

@@ -1,4 +1,5 @@
-import { createEventSourceConnection } from "./event_source_connection.js"
+import { createEventSourceConnection } from "@jsenv/core/src/helpers/event_source/event_source.js"
+import { urlHotMetas } from "../../../import_meta_hot/client/import_meta_hot.js"
 import {
   isAutoreloadEnabled,
   setAutoreloadPreference,
@@ -9,7 +10,6 @@ import {
   reloadDOMNodesUsingUrl,
   reloadJsImport,
 } from "./reload.js"
-import { urlHotMetas } from "./import_meta_hot.js"
 
 const reloadMessages = []
 const reloadMessagesSignal = { onchange: () => {} }
@@ -26,13 +26,16 @@ const dequeue = async () => {
   const callbacks = pendingCallbacks.slice()
   pendingCallbacks = []
   running = true
-  await callbacks.reduce(async (previous, callback) => {
-    await previous
-    await callback()
-  }, Promise.resolve())
-  running = false
-  if (pendingCallbacks.length) {
-    dequeue()
+  try {
+    await callbacks.reduce(async (previous, callback) => {
+      await previous
+      await callback()
+    }, Promise.resolve())
+  } finally {
+    running = false
+    if (pendingCallbacks.length) {
+      dequeue()
+    }
   }
 }
 
@@ -93,11 +96,15 @@ const applyHotReload = async ({ hotInstructions }) => {
       // if (!urlHotMeta) {return }
 
       if (type === "prune") {
-        console.group(`[jsenv] prune: ${boundary} (inside ${acceptedBy})`)
+        console.groupCollapsed(
+          `[jsenv] prune: ${boundary} (inside ${acceptedBy})`,
+        )
       } else if (acceptedBy === boundary) {
-        console.group(`[jsenv] hot reloading: ${boundary}`)
+        console.groupCollapsed(`[jsenv] hot reloading: ${boundary}`)
       } else {
-        console.group(`[jsenv] hot reloading: ${acceptedBy} inside ${boundary}`)
+        console.groupCollapsed(
+          `[jsenv] hot reloading: ${acceptedBy} inside ${boundary}`,
+        )
       }
       if (urlHotMeta && urlHotMeta.disposeCallback) {
         console.log(`call dispose callback`)

package/src/{dev/plugins/autoreload → plugins/autoreload/dev_sse}/client/reload.js

@@ -1,4 +1,7 @@
-import { htmlAttributeSrcSet } from "@jsenv/utils/html_ast/html_attribute_src_set.js"
+import {
+  parseSrcSet,
+  stringifySrcSet,
+} from "@jsenv/ast/src/html/html_src_set.js"
 
 import { injectQuery, compareTwoUrlPaths } from "./url_helpers.js"
 
@@ -57,7 +60,7 @@ export const reloadDOMNodesUsingUrl = (urlToReload) => {
     visitNodeAttributeAsUrl(img, "src")
     const srcset = img.srcset
     if (srcset) {
-      const srcCandidates = htmlAttributeSrcSet.parse(srcset)
+      const srcCandidates = parseSrcSet(srcset)
       srcCandidates.forEach((srcCandidate) => {
         const url = new URL(srcCandidate.specifier, `${window.location.href}`)
         if (shouldReloadUrl(url)) {
@@ -65,7 +68,7 @@ export const reloadDOMNodesUsingUrl = (urlToReload) => {
         }
       })
       mutations.push(() => {
-        img.srcset = htmlAttributeSrcSet.stringify(srcCandidates)
+        img.srcset = stringifySrcSet(srcCandidates)
       })
     }
   })

package/src/plugins/autoreload/dev_sse/jsenv_plugin_dev_sse_client.js

@@ -0,0 +1,41 @@
+import {
+  parseHtmlString,
+  stringifyHtmlAst,
+  injectScriptNodeAsEarlyAsPossible,
+  createHtmlNode,
+} from "@jsenv/ast"
+
+export const jsenvPluginDevSSEClient = () => {
+  const eventSourceClientFileUrl = new URL(
+    "./client/event_source_client.js",
+    import.meta.url,
+  ).href
+
+  return {
+    name: "jsenv:dev_sse_client",
+    appliesDuring: { dev: true },
+    transformUrlContent: {
+      html: (htmlUrlInfo, context) => {
+        const htmlAst = parseHtmlString(htmlUrlInfo.content)
+        const [eventSourceClientReference] = context.referenceUtils.inject({
+          type: "script_src",
+          expectedType: "js_module",
+          specifier: eventSourceClientFileUrl,
+        })
+        injectScriptNodeAsEarlyAsPossible(
+          htmlAst,
+          createHtmlNode({
+            "tagName": "script",
+            "type": "module",
+            "src": eventSourceClientReference.generatedSpecifier,
+            "injected-by": "jsenv:dev_sse_client",
+          }),
+        )
+        const htmlModified = stringifyHtmlAst(htmlAst)
+        return {
+          content: htmlModified,
+        }
+      },
+    },
+  }
+}