@jsenv/core 27.0.0-alpha.4 → 27.0.0-alpha.42

package/src/build/build_urls_generator.js

@@ -4,24 +4,33 @@ import { memoizeByUrl } from "@jsenv/utils/memoize/memoize_by_url.js"
 
 export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
   const cache = {}
-  const generate = memoizeByUrl((url, urlInfo, parentUrlInfo) => {
-    const directoryPath = determineDirectoryPath(urlInfo, parentUrlInfo)
-    let name = urlToFilename(url)
-    const { searchParams } = new URL(url)
-    if (
-      searchParams.has("css_module") ||
-      searchParams.has("json_module") ||
-      searchParams.has("text_module")
-    ) {
-      name = `${name}.js`
+
+  const getUrlName = (url, urlInfo) => {
+    if (!urlInfo) {
+      return urlToFilename(url)
+    }
+    if (urlInfo.filename) {
+      return urlInfo.filename
     }
+    return urlToFilename(url)
+  }
+
+  const generate = memoizeByUrl((url, { urlInfo, parentUrlInfo }) => {
+    const directoryPath = determineDirectoryPath({
+      urlInfo,
+      parentUrlInfo,
+    })
     let names = cache[directoryPath]
     if (!names) {
       names = []
       cache[directoryPath] = names
     }
-
-    let nameCandidate = name
+    const urlObject = new URL(url)
+    let { search, hash } = urlObject
+    let name = getUrlName(url, urlInfo)
+    let [basename, extension] = splitFileExtension(name)
+    extension = extensionMappings[extension] || extension
+    let nameCandidate = `${basename}${extension}` // reconstruct name in case extension was normalized
     let integer = 1
     // eslint-disable-next-line no-constant-condition
     while (true) {
@@ -30,9 +39,9 @@ export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
         break
       }
       integer++
-      nameCandidate = `${name}${integer}`
+      nameCandidate = `${basename}${integer}${extension}`
     }
-    return `${buildDirectoryUrl}${directoryPath}${nameCandidate}`
+    return `${buildDirectoryUrl}${directoryPath}${nameCandidate}${search}${hash}`
   })
 
   return {
@@ -40,12 +49,37 @@ export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
   }
 }
 
-const determineDirectoryPath = (urlInfo, parentUrlInfo) => {
+// It's best to generate files with an extension representing what is inside the file
+// and after build js files contains solely js (js or typescript is gone).
+// This way a static file server is already configured to server the correct content-type
+// (otherwise one would have to configure that ".jsx" is "text/javascript")
+// To keep in mind: if you have "user.jsx" and "user.js" AND both file are not bundled
+// you end up with "dist/js/user.js" and "dist/js/user2.js"
+const extensionMappings = {
+  ".ts": ".js",
+  ".jsx": ".js",
+  ".tsx": ".js",
+}
+
+const splitFileExtension = (filename) => {
+  const dotLastIndex = filename.lastIndexOf(".")
+  if (dotLastIndex === -1) {
+    return [filename, ""]
+  }
+  return [filename.slice(0, dotLastIndex), filename.slice(dotLastIndex)]
+}
+
+const determineDirectoryPath = ({ urlInfo, parentUrlInfo }) => {
   if (urlInfo.isInline) {
-    const parentDirectoryPath = determineDirectoryPath(parentUrlInfo)
+    const parentDirectoryPath = determineDirectoryPath({
+      urlInfo: parentUrlInfo,
+    })
     return parentDirectoryPath
   }
-  if (urlInfo.data.isEntryPoint) {
+  if (urlInfo.data.isEntryPoint || urlInfo.data.isWebWorkerEntryPoint) {
+    return ""
+  }
+  if (urlInfo.type === "importmap") {
     return ""
   }
   if (urlInfo.type === "html") {
@@ -55,10 +89,10 @@ const determineDirectoryPath = (urlInfo, parentUrlInfo) => {
     return "css/"
   }
   if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
-    if (urlInfo.subtype === "service_worker") {
-      return ""
-    }
     return "js/"
   }
-  return "assets/"
+  if (urlInfo.type === "json") {
+    return "json/"
+  }
+  return "other/"
 }

package/src/build/graph_utils.js

@@ -0,0 +1,31 @@
+export const GRAPH = {
+  map: (graph, callback) => {
+    return Object.keys(graph.urlInfos).map((url) => {
+      return callback(graph.urlInfos[url])
+    })
+  },
+
+  forEach: (graph, callback) => {
+    Object.keys(graph.urlInfos).forEach((url) => {
+      callback(graph.urlInfos[url], url)
+    })
+  },
+
+  filter: (graph, callback) => {
+    const urlInfos = []
+    Object.keys(graph.urlInfos).forEach((url) => {
+      const urlInfo = graph.urlInfos[url]
+      if (callback(urlInfo)) {
+        urlInfos.push(urlInfo)
+      }
+    })
+    return urlInfos
+  },
+
+  find: (graph, callback) => {
+    const urlFound = Object.keys(graph.urlInfos).find((url) => {
+      return callback(graph.urlInfos[url])
+    })
+    return graph.urlInfos[urlFound]
+  },
+}

package/src/build/{inject_version_mappings.js → inject_global_version_mappings.js}

@@ -8,10 +8,27 @@ import {
 } from "@jsenv/utils/html_ast/html_ast.js"
 import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
 
-export const injectVersionMappings = async (
-  urlInfo,
-  { kitchen, versionMappings },
-) => {
+import { GRAPH } from "./graph_utils.js"
+
+export const injectGlobalVersionMapping = async ({
+  finalGraphKitchen,
+  finalGraph,
+  versionMappings,
+}) => {
+  await Promise.all(
+    GRAPH.map(finalGraph, async (urlInfo) => {
+      if (urlInfo.data.isEntryPoint || urlInfo.data.isWebWorkerEntryPoint) {
+        await injectVersionMappings({
+          urlInfo,
+          kitchen: finalGraphKitchen,
+          versionMappings,
+        })
+      }
+    }),
+  )
+}
+
+const injectVersionMappings = async ({ urlInfo, kitchen, versionMappings }) => {
   const injector = injectors[urlInfo.type]
   if (injector) {
     const { content, sourcemap } = injector(urlInfo, { versionMappings })
@@ -22,6 +39,12 @@ export const injectVersionMappings = async (
   }
 }
 
+const jsInjector = (urlInfo, { versionMappings }) => {
+  const magicSource = createMagicSource(urlInfo.content)
+  magicSource.prepend(generateClientCodeForVersionMappings(versionMappings))
+  return magicSource.toContentAndSourcemap()
+}
+
 const injectors = {
   html: (urlInfo, { versionMappings }) => {
     // ideally we would inject an importmap but browser support is too low
@@ -39,24 +62,19 @@ const injectors = {
       }),
     )
     return {
-      content: stringifyHtmlAst(htmlAst, {
-        removeOriginalPositionAttributes: true,
-      }),
+      content: stringifyHtmlAst(htmlAst),
     }
   },
-  js_module: (urlInfo, { versionMappings }) => {
-    const magicSource = createMagicSource(urlInfo.content)
-    magicSource.prepend(generateClientCodeForVersionMappings(versionMappings))
-    return magicSource.toContentAndSourcemap()
-  },
+  js_classic: jsInjector,
+  js_module: jsInjector,
 }
 
 const generateClientCodeForVersionMappings = (versionMappings) => {
   return `
-var __versionMappings__ = ${JSON.stringify(versionMappings, null, "  ")}
-var __envGlobal__ = typeof self === 'undefined' ? global : self
+var __versionMappings__ = ${JSON.stringify(versionMappings, null, "  ")};
+var __envGlobal__ = typeof self === 'undefined' ? global : self;
 __envGlobal__.__v__ = function (specifier) {
   return __versionMappings__[specifier] || specifier
-}
+};
 `
 }

package/src/build/inject_service_worker_urls.js

@@ -0,0 +1,79 @@
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+import { createVersionGenerator } from "@jsenv/utils/versioning/version_generator.js"
+
+import { GRAPH } from "./graph_utils.js"
+
+export const injectServiceWorkerUrls = async ({
+  finalGraph,
+  finalGraphKitchen,
+  lineBreakNormalization,
+}) => {
+  const serviceWorkerEntryUrlInfos = GRAPH.filter(
+    finalGraph,
+    (finalUrlInfo) => {
+      return (
+        finalUrlInfo.subtype === "service_worker" &&
+        finalUrlInfo.data.isWebWorkerEntryPoint
+      )
+    },
+  )
+  if (serviceWorkerEntryUrlInfos.length === 0) {
+    return
+  }
+  const serviceWorkerUrls = {}
+  GRAPH.forEach(finalGraph, (urlInfo) => {
+    if (urlInfo.isInline || urlInfo.external) {
+      return
+    }
+    if (!urlInfo.url.startsWith("file:")) {
+      return
+    }
+    if (urlInfo.data.buildUrlIsVersioned) {
+      serviceWorkerUrls[urlInfo.data.buildUrlSpecifier] = {
+        versioned: true,
+      }
+      return
+    }
+    if (!urlInfo.data.version) {
+      // when url is not versioned we compute a "version" for that url anyway
+      // so that service worker source still changes and navigator
+      // detect there is a change
+      const versionGenerator = createVersionGenerator()
+      versionGenerator.augmentWithContent({
+        content: urlInfo.content,
+        contentType: urlInfo.contentType,
+        lineBreakNormalization,
+      })
+      const version = versionGenerator.generate()
+      urlInfo.data.version = version
+    }
+    serviceWorkerUrls[urlInfo.data.buildUrlSpecifier] = {
+      versioned: false,
+      version: urlInfo.data.version,
+    }
+  })
+  await Promise.all(
+    serviceWorkerEntryUrlInfos.map(async (serviceWorkerEntryUrlInfo) => {
+      const magicSource = createMagicSource(serviceWorkerEntryUrlInfo.content)
+      const urlsWithoutSelf = {
+        ...serviceWorkerUrls,
+      }
+      delete urlsWithoutSelf[serviceWorkerEntryUrlInfo.data.buildUrlSpecifier]
+      magicSource.prepend(generateClientCode(urlsWithoutSelf))
+      const { content, sourcemap } = magicSource.toContentAndSourcemap()
+      await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
+        serviceWorkerEntryUrlInfo,
+        {
+          content,
+          sourcemap,
+        },
+      )
+    }),
+  )
+}
+
+const generateClientCode = (serviceWorkerUrls) => {
+  return `
+self.serviceWorkerUrls = ${JSON.stringify(serviceWorkerUrls, null, "  ")};
+`
+}

package/src/build/resync_ressource_hints.js

@@ -0,0 +1,68 @@
+/*
+ * Update <link rel="preload"> and friends after build (once we know everything)
+ *
+ * - Used to remove ressource hint targeting an url that is no longer used:
+ *   - Happens because of import assertions transpilation (file is inlined into JS)
+ */
+
+import {
+  parseHtmlString,
+  visitHtmlAst,
+  stringifyHtmlAst,
+  getHtmlNodeAttributeByName,
+  removeHtmlNode,
+} from "@jsenv/utils/html_ast/html_ast.js"
+
+import { GRAPH } from "./graph_utils.js"
+
+export const resyncRessourceHints = async ({
+  finalGraphKitchen,
+  finalGraph,
+  buildUrls,
+}) => {
+  const ressourceHintActions = []
+  GRAPH.forEach(finalGraph, (urlInfo) => {
+    if (urlInfo.type !== "html") {
+      return
+    }
+    ressourceHintActions.push(async () => {
+      const htmlAst = parseHtmlString(urlInfo.content, {
+        storeOriginalPositions: false,
+      })
+      const visitLinkWithHref = (linkNode, hrefAttribute) => {
+        const href = hrefAttribute.value
+        if (!href || href.startsWith("data:")) {
+          return
+        }
+        const buildUrl = buildUrls[href]
+        const urlInfo = finalGraph.getUrlInfo(buildUrl)
+        if (!urlInfo) {
+          return
+        }
+        if (urlInfo.dependents.size === 0) {
+          removeHtmlNode(linkNode)
+          return
+        }
+      }
+      visitHtmlAst(htmlAst, (node) => {
+        if (node.nodeName !== "link") {
+          return
+        }
+        const hrefAttribute = getHtmlNodeAttributeByName(node, "href")
+        if (!hrefAttribute) {
+          return
+        }
+        visitLinkWithHref(node, hrefAttribute)
+      })
+      await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
+        urlInfo,
+        {
+          content: stringifyHtmlAst(htmlAst),
+        },
+      )
+    })
+  })
+  await Promise.all(
+    ressourceHintActions.map((ressourceHintAction) => ressourceHintAction()),
+  )
+}

package/src/build/start_build_server.js

@@ -0,0 +1,205 @@
+/*
+ * startBuildServer is mean to interact with the build files;
+ * files that will be deployed to production server(s).
+ * We want to be as close as possible from the production in order to:
+ * - run lighthouse
+ * - run an automated test tool such as cypress, playwright
+ * - see exactly how build file behaves (debug, measure perf, etc)
+ * For these reasons "startBuildServer" must be as close as possible from a static file server.
+ * It is not meant to provide a nice developper experience: this is the role "startDevServer".
+ *
+ * Conclusion:
+ * "startBuildServer" must be as close as possible from a static file server because
+ * we want to be in the user shoes and we should not alter build files.
+ */
+
+import {
+  jsenvAccessControlAllowedHeaders,
+  startServer,
+  pluginServerTiming,
+  pluginRequestWaitingCheck,
+  pluginCORS,
+  fetchFileSystem,
+} from "@jsenv/server"
+import { assertAndNormalizeDirectoryUrl } from "@jsenv/filesystem"
+import { createLogger } from "@jsenv/logger"
+import { Abort } from "@jsenv/abort"
+
+import { initProcessAutorestart } from "@jsenv/utils/file_watcher/process_auto_restart.js"
+import { createTaskLog } from "@jsenv/utils/logs/task_log.js"
+import { watchFiles } from "@jsenv/utils/file_watcher/file_watcher.js"
+import { executeCommand } from "@jsenv/utils/command/command.js"
+
+export const startBuildServer = async ({
+  signal = new AbortController().signal,
+  handleSIGINT = true,
+  logLevel,
+  buildCommandLogLevel = "warn",
+  protocol = "http",
+  http2,
+  certificate,
+  privateKey,
+  listenAnyIp,
+  ip,
+  port = 9779,
+
+  rootDirectoryUrl,
+  buildDirectoryUrl,
+  buildCommand,
+  mainBuildFileUrl = "/index.html",
+  watchedFilePatterns,
+  cooldownBetweenFileEvents,
+  autorestart,
+}) => {
+  rootDirectoryUrl = assertAndNormalizeDirectoryUrl(rootDirectoryUrl)
+  buildDirectoryUrl = assertAndNormalizeDirectoryUrl(buildDirectoryUrl)
+
+  const autorestartProcess = await initProcessAutorestart({
+    signal,
+    handleSIGINT,
+    ...(autorestart
+      ? {
+          enabled: true,
+          logLevel: autorestart.logLevel,
+          urlToRestart: autorestart.url,
+          urlsToWatch: [
+            ...(autorestart.urlsToWatch || []),
+            new URL("package.json", rootDirectoryUrl),
+            new URL("jsenv.config.mjs", rootDirectoryUrl),
+          ],
+        }
+      : {
+          enabled: false,
+        }),
+  })
+  if (autorestartProcess.isPrimary) {
+    return {
+      origin: `${protocol}://127.0.0.1:${port}`,
+      stop: () => {
+        autorestartProcess.stop()
+      },
+    }
+  }
+  signal = autorestartProcess.signal
+  const logger = createLogger({ logLevel })
+
+  let buildPromise
+  let buildAbortController
+  const runBuild = async () => {
+    buildAbortController = new AbortController()
+    const buildOperation = Abort.startOperation()
+    buildOperation.addAbortSignal(signal)
+    buildOperation.addAbortSignal(buildAbortController.signal)
+    const buildTask = createTaskLog(logger, `execute build command`)
+    buildPromise = executeCommand(buildCommand, {
+      cwd: rootDirectoryUrl,
+      logLevel: buildCommandLogLevel,
+      signal: buildOperation.signal,
+    })
+    try {
+      await buildPromise
+      buildTask.done()
+    } catch (e) {
+      if (e.code === "ABORT_ERR") {
+        buildTask.fail(`execute build command aborted`)
+      } else {
+        buildTask.fail()
+        throw e
+      }
+    }
+  }
+
+  const startServerTask = createTaskLog(logger, "start build server")
+  const server = await startServer({
+    signal,
+    stopOnExit: false,
+    stopOnSIGINT: false,
+    stopOnInternalError: false,
+    keepProcessAlive: true,
+    logLevel,
+    startLog: false,
+
+    protocol,
+    http2,
+    certificate,
+    privateKey,
+    listenAnyIp,
+    ip,
+    port,
+    plugins: {
+      ...pluginCORS({
+        accessControlAllowRequestOrigin: true,
+        accessControlAllowRequestMethod: true,
+        accessControlAllowRequestHeaders: true,
+        accessControlAllowedRequestHeaders: [
+          ...jsenvAccessControlAllowedHeaders,
+          "x-jsenv-execution-id",
+        ],
+        accessControlAllowCredentials: true,
+      }),
+      ...pluginServerTiming(),
+      ...pluginRequestWaitingCheck({
+        requestWaitingMs: 60 * 1000,
+      }),
+    },
+    sendErrorDetails: true,
+    requestToResponse: async (request) => {
+      await buildPromise
+      const urlIsVersioned = new URL(
+        request.ressource,
+        request.origin,
+      ).searchParams.has("v")
+      if (mainBuildFileUrl && request.ressource === "/") {
+        request = {
+          ...request,
+          ressource: mainBuildFileUrl,
+        }
+      }
+      return fetchFileSystem(
+        new URL(request.ressource.slice(1), buildDirectoryUrl),
+        {
+          headers: request.headers,
+          cacheControl: urlIsVersioned
+            ? `private,max-age=${SECONDS_IN_30_DAYS},immutable`
+            : "private,max-age=0,must-revalidate",
+          etagEnabled: true,
+          compressionEnabled: !request.pathname.endsWith(".mp4"),
+          rootDirectoryUrl: buildDirectoryUrl,
+          canReadDirectory: true,
+        },
+      )
+    },
+  })
+  startServerTask.done()
+  logger.info(``)
+  Object.keys(server.origins).forEach((key) => {
+    logger.info(`- ${server.origins[key]}`)
+  })
+  logger.info(``)
+
+  const unregisterDirectoryLifecyle = watchFiles({
+    rootDirectoryUrl,
+    watchedFilePatterns,
+    cooldownBetweenFileEvents,
+    fileChangeCallback: ({ url, event }) => {
+      buildAbortController.abort()
+      // setTimeout is to ensure the abortController.abort() above
+      // is properly taken into account so that logs about abort comes first
+      // then logs about re-running the build happens
+      setTimeout(() => {
+        logger.info(`${url.slice(rootDirectoryUrl.length)} ${event} -> rebuild`)
+        runBuild()
+      })
+    },
+  })
+  signal.addEventListener("abort", () => {
+    unregisterDirectoryLifecyle()
+  })
+  runBuild()
+  return {
+    origin: server.origin,
+    stop: () => server.stop(),
+  }
+}
+
+const SECONDS_IN_30_DAYS = 60 * 60 * 24 * 30

package/src/dev/plugins/explorer/jsenv_plugin_explorer.js

@@ -1,8 +1,8 @@
 import { readFileSync } from "node:fs"
-import { urlToContentType } from "@jsenv/server"
 import { collectFiles, normalizeStructuredMetaMap } from "@jsenv/filesystem"
 
 import { DataUrl } from "@jsenv/utils/urls/data_url.js"
+import { CONTENT_TYPE } from "@jsenv/utils/content_type/content_type.js"
 
 export const jsenvPluginExplorer = ({ groups }) => {
   const htmlClientFileUrl = new URL("./client/explorer.html", import.meta.url)
@@ -43,7 +43,7 @@ export const jsenvPluginExplorer = ({ groups }) => {
       html = html.replace(
         "FAVICON_HREF",
         DataUrl.stringify({
-          contentType: urlToContentType(faviconClientFileUrl),
+          contentType: CONTENT_TYPE.fromUrlExtension(faviconClientFileUrl),
           base64Flag: true,
           data: readFileSync(new URL(faviconClientFileUrl)).toString("base64"),
         }),

package/src/dev/plugins/toolbar/jsenv_plugin_toolbar.js

@@ -20,7 +20,7 @@ export const jsenvPluginToolbar = ({ logs = false } = {}) => {
     appliesDuring: {
       dev: true,
     },
-    transform: {
+    transformUrlContent: {
       html: ({ url, content }, { referenceUtils }) => {
         if (url === toolbarHtmlClientFileUrl) {
           return null
@@ -28,10 +28,12 @@ export const jsenvPluginToolbar = ({ logs = false } = {}) => {
         const htmlAst = parseHtmlString(content)
         const [toolbarInjectorReference] = referenceUtils.inject({
           type: "js_import_export",
+          expectedType: "js_module",
           specifier: toolbarInjectorClientFileUrl,
         })
         const [toolbarClientFileReference] = referenceUtils.inject({
           type: "iframe_src",
+          expectedType: "html",
           specifier: toolbarHtmlClientFileUrl,
         })
         injectScriptAsEarlyAsPossible(