@jsenv/core 27.0.0-alpha.9 → 27.0.0-alpha.92

package/src/build/build_urls_generator.js

@@ -1,28 +1,36 @@
-import { urlToFilename } from "@jsenv/filesystem"
-
-import { memoizeByUrl } from "@jsenv/utils/memoize/memoize_by_url.js"
+import { urlToFilename, urlToRelativeUrl } from "@jsenv/urls"
+import { memoizeByFirstArgument } from "@jsenv/utils/src/memoize/memoize_by_first_argument.js"
 
 export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
   const cache = {}
-  const generate = memoizeByUrl((url, urlInfo, parentUrlInfo) => {
-    const directoryPath = determineDirectoryPath(urlInfo, parentUrlInfo)
+
+  const getUrlName = (url, urlInfo) => {
+    if (!urlInfo) {
+      return urlToFilename(url)
+    }
+    if (urlInfo.filename) {
+      return urlInfo.filename
+    }
+    return urlToFilename(url)
+  }
+
+  const generate = memoizeByFirstArgument((url, { urlInfo, parentUrlInfo }) => {
+    const directoryPath = determineDirectoryPath({
+      buildDirectoryUrl,
+      urlInfo,
+      parentUrlInfo,
+    })
     let names = cache[directoryPath]
     if (!names) {
       names = []
       cache[directoryPath] = names
     }
-
-    let name = urlToFilename(url)
-    const { searchParams } = new URL(url)
-    if (
-      searchParams.has("css_module") ||
-      searchParams.has("json_module") ||
-      searchParams.has("text_module")
-    ) {
-      name = `${name}.js`
-    }
-    const [basename, extension] = splitFileExtension(name)
-    let nameCandidate = name
+    const urlObject = new URL(url)
+    let { search, hash } = urlObject
+    let name = getUrlName(url, urlInfo)
+    let [basename, extension] = splitFileExtension(name)
+    extension = extensionMappings[extension] || extension
+    let nameCandidate = `${basename}${extension}` // reconstruct name in case extension was normalized
     let integer = 1
     // eslint-disable-next-line no-constant-condition
     while (true) {
@@ -33,7 +41,7 @@ export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
       integer++
       nameCandidate = `${basename}${integer}${extension}`
     }
-    return `${buildDirectoryUrl}${directoryPath}${nameCandidate}`
+    return `${buildDirectoryUrl}${directoryPath}${nameCandidate}${search}${hash}`
   })
 
   return {
@@ -41,6 +49,18 @@ export const createBuilUrlsGenerator = ({ buildDirectoryUrl }) => {
   }
 }
 
+// It's best to generate files with an extension representing what is inside the file
+// and after build js files contains solely js (js or typescript is gone).
+// This way a static file server is already configured to server the correct content-type
+// (otherwise one would have to configure that ".jsx" is "text/javascript")
+// To keep in mind: if you have "user.jsx" and "user.js" AND both file are not bundled
+// you end up with "dist/js/user.js" and "dist/js/user2.js"
+const extensionMappings = {
+  ".jsx": ".js",
+  ".ts": ".js",
+  ".tsx": ".js",
+}
+
 const splitFileExtension = (filename) => {
   const dotLastIndex = filename.lastIndexOf(".")
   if (dotLastIndex === -1) {
@@ -49,12 +69,32 @@ const splitFileExtension = (filename) => {
   return [filename.slice(0, dotLastIndex), filename.slice(dotLastIndex)]
 }
 
-const determineDirectoryPath = (urlInfo, parentUrlInfo) => {
+const determineDirectoryPath = ({
+  buildDirectoryUrl,
+  urlInfo,
+  parentUrlInfo,
+}) => {
+  if (urlInfo.type === "directory") {
+    return ""
+  }
+  if (parentUrlInfo && parentUrlInfo.type === "directory") {
+    const parentDirectoryPath = urlToRelativeUrl(
+      parentUrlInfo.url,
+      buildDirectoryUrl,
+    )
+    return parentDirectoryPath
+  }
   if (urlInfo.isInline) {
-    const parentDirectoryPath = determineDirectoryPath(parentUrlInfo)
+    const parentDirectoryPath = determineDirectoryPath({
+      buildDirectoryUrl,
+      urlInfo: parentUrlInfo,
+    })
     return parentDirectoryPath
   }
-  if (urlInfo.data.isEntryPoint) {
+  if (urlInfo.data.isEntryPoint || urlInfo.data.isWebWorkerEntryPoint) {
+    return ""
+  }
+  if (urlInfo.type === "importmap") {
     return ""
   }
   if (urlInfo.type === "html") {
@@ -64,10 +104,10 @@ const determineDirectoryPath = (urlInfo, parentUrlInfo) => {
     return "css/"
   }
   if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
-    if (urlInfo.subtype === "service_worker") {
-      return ""
-    }
     return "js/"
   }
-  return "assets/"
+  if (urlInfo.type === "json") {
+    return "json/"
+  }
+  return "other/"
 }

package/src/build/graph_utils.js

@@ -0,0 +1,31 @@
+export const GRAPH = {
+  map: (graph, callback) => {
+    return Object.keys(graph.urlInfos).map((url) => {
+      return callback(graph.urlInfos[url])
+    })
+  },
+
+  forEach: (graph, callback) => {
+    Object.keys(graph.urlInfos).forEach((url) => {
+      callback(graph.urlInfos[url], url)
+    })
+  },
+
+  filter: (graph, callback) => {
+    const urlInfos = []
+    Object.keys(graph.urlInfos).forEach((url) => {
+      const urlInfo = graph.urlInfos[url]
+      if (callback(urlInfo)) {
+        urlInfos.push(urlInfo)
+      }
+    })
+    return urlInfos
+  },
+
+  find: (graph, callback) => {
+    const urlFound = Object.keys(graph.urlInfos).find((url) => {
+      return callback(graph.urlInfos[url])
+    })
+    return graph.urlInfos[urlFound]
+  },
+}

package/src/build/inject_global_version_mappings.js

@@ -0,0 +1,93 @@
+// https://bundlers.tooling.report/hashing/avoid-cascade/
+
+import { createMagicSource } from "@jsenv/sourcemap"
+import {
+  parseHtmlString,
+  injectScriptNodeAsEarlyAsPossible,
+  createHtmlNode,
+  stringifyHtmlAst,
+} from "@jsenv/ast"
+
+import { GRAPH } from "./graph_utils.js"
+
+export const injectGlobalVersionMapping = async ({
+  finalGraphKitchen,
+  finalGraph,
+  versionMappings,
+}) => {
+  await Promise.all(
+    GRAPH.map(finalGraph, async (urlInfo) => {
+      if (urlInfo.data.isEntryPoint || urlInfo.data.isWebWorkerEntryPoint) {
+        await injectVersionMappings({
+          urlInfo,
+          kitchen: finalGraphKitchen,
+          versionMappings,
+        })
+      }
+    }),
+  )
+}
+
+const injectVersionMappings = async ({ urlInfo, kitchen, versionMappings }) => {
+  const injector = injectors[urlInfo.type]
+  if (injector) {
+    const { content, sourcemap } = injector(urlInfo, { versionMappings })
+    await kitchen.urlInfoTransformer.applyFinalTransformations(urlInfo, {
+      content,
+      sourcemap,
+    })
+  }
+}
+
+const jsInjector = (urlInfo, { versionMappings }) => {
+  const magicSource = createMagicSource(urlInfo.content)
+  magicSource.prepend(
+    generateClientCodeForVersionMappings(versionMappings, {
+      globalName: urlInfo.data.isWebWorkerEntryPoint ? "self" : "window",
+    }),
+  )
+  return magicSource.toContentAndSourcemap()
+}
+
+const injectors = {
+  html: (urlInfo, { versionMappings }) => {
+    // ideally we would inject an importmap but browser support is too low
+    // (even worse for worker/service worker)
+    // so for now we inject code into entry points
+    const htmlAst = parseHtmlString(urlInfo.content, {
+      storeOriginalPositions: false,
+    })
+    injectScriptNodeAsEarlyAsPossible(
+      htmlAst,
+      createHtmlNode({
+        "tagName": "script",
+        "textContent": generateClientCodeForVersionMappings(versionMappings, {
+          globalName: "window",
+        }),
+        "injected-by": "jsenv:versioning",
+      }),
+    )
+    return {
+      content: stringifyHtmlAst(htmlAst),
+    }
+  },
+  js_classic: jsInjector,
+  js_module: jsInjector,
+}
+
+const generateClientCodeForVersionMappings = (
+  versionMappings,
+  { globalName },
+) => {
+  return `
+;(function() {
+
+var __versionMappings__ = ${JSON.stringify(versionMappings, null, "  ")};
+${globalName}.__v__ = function (specifier) {
+  return __versionMappings__[specifier] || specifier
+};
+
+})();
+
+`
+}

package/src/build/inject_service_worker_urls.js

@@ -0,0 +1,79 @@
+import { createMagicSource } from "@jsenv/sourcemap"
+
+import { GRAPH } from "./graph_utils.js"
+import { createVersionGenerator } from "./version_generator.js"
+
+export const injectServiceWorkerUrls = async ({
+  finalGraph,
+  finalGraphKitchen,
+  lineBreakNormalization,
+}) => {
+  const serviceWorkerEntryUrlInfos = GRAPH.filter(
+    finalGraph,
+    (finalUrlInfo) => {
+      return (
+        finalUrlInfo.subtype === "service_worker" &&
+        finalUrlInfo.data.isWebWorkerEntryPoint
+      )
+    },
+  )
+  if (serviceWorkerEntryUrlInfos.length === 0) {
+    return
+  }
+  const serviceWorkerUrls = {}
+  GRAPH.forEach(finalGraph, (urlInfo) => {
+    if (urlInfo.isInline || !urlInfo.shouldHandle) {
+      return
+    }
+    if (!urlInfo.url.startsWith("file:")) {
+      return
+    }
+    if (urlInfo.data.buildUrlIsVersioned) {
+      serviceWorkerUrls[urlInfo.data.buildUrlSpecifier] = {
+        versioned: true,
+      }
+      return
+    }
+    if (!urlInfo.data.version) {
+      // when url is not versioned we compute a "version" for that url anyway
+      // so that service worker source still changes and navigator
+      // detect there is a change
+      const versionGenerator = createVersionGenerator()
+      versionGenerator.augmentWithContent({
+        content: urlInfo.content,
+        contentType: urlInfo.contentType,
+        lineBreakNormalization,
+      })
+      const version = versionGenerator.generate()
+      urlInfo.data.version = version
+    }
+    serviceWorkerUrls[urlInfo.data.buildUrlSpecifier] = {
+      versioned: false,
+      version: urlInfo.data.version,
+    }
+  })
+  await Promise.all(
+    serviceWorkerEntryUrlInfos.map(async (serviceWorkerEntryUrlInfo) => {
+      const magicSource = createMagicSource(serviceWorkerEntryUrlInfo.content)
+      const urlsWithoutSelf = {
+        ...serviceWorkerUrls,
+      }
+      delete urlsWithoutSelf[serviceWorkerEntryUrlInfo.data.buildUrlSpecifier]
+      magicSource.prepend(generateClientCode(urlsWithoutSelf))
+      const { content, sourcemap } = magicSource.toContentAndSourcemap()
+      await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
+        serviceWorkerEntryUrlInfo,
+        {
+          content,
+          sourcemap,
+        },
+      )
+    }),
+  )
+}
+
+const generateClientCode = (serviceWorkerUrls) => {
+  return `
+self.serviceWorkerUrls = ${JSON.stringify(serviceWorkerUrls, null, "  ")};
+`
+}

package/src/build/resync_ressource_hints.js

@@ -0,0 +1,114 @@
+/*
+ * Update <link rel="preload"> and friends after build (once we know everything)
+ *
+ * - Used to remove ressource hint targeting an url that is no longer used:
+ *   - Happens because of import assertions transpilation (file is inlined into JS)
+ */
+
+import {
+  parseHtmlString,
+  visitHtmlNodes,
+  getHtmlNodeAttribute,
+  setHtmlNodeAttributes,
+  removeHtmlNode,
+  stringifyHtmlAst,
+} from "@jsenv/ast"
+
+import { GRAPH } from "./graph_utils.js"
+
+export const resyncRessourceHints = async ({
+  logger,
+  finalGraphKitchen,
+  finalGraph,
+  rawUrls,
+  postBuildRedirections,
+}) => {
+  const ressourceHintActions = []
+  GRAPH.forEach(finalGraph, (urlInfo) => {
+    if (urlInfo.type !== "html") {
+      return
+    }
+    ressourceHintActions.push(async () => {
+      const htmlAst = parseHtmlString(urlInfo.content, {
+        storeOriginalPositions: false,
+      })
+      const actions = []
+      const visitLinkWithHref = (linkNode, href) => {
+        if (!href || href.startsWith("data:")) {
+          return
+        }
+        const rel = getHtmlNodeAttribute(linkNode, "rel")
+        const isRessourceHint = [
+          "preconnect",
+          "dns-prefetch",
+          "prefetch",
+          "preload",
+          "modulepreload",
+        ].includes(rel)
+        if (!isRessourceHint) {
+          return
+        }
+
+        let buildUrl
+        for (const key of Object.keys(rawUrls)) {
+          if (rawUrls[key] === href) {
+            buildUrl = key
+            break
+          }
+        }
+        if (!buildUrl) {
+          logger.warn(`remove ressource hint because cannot find "${href}"`)
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
+          return
+        }
+        buildUrl = postBuildRedirections[buildUrl] || buildUrl
+        const urlInfo = finalGraph.getUrlInfo(buildUrl)
+        if (!urlInfo) {
+          logger.warn(
+            `remove ressource hint because cannot find "${buildUrl}" in the graph`,
+          )
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
+          return
+        }
+        if (urlInfo.dependents.size === 0) {
+          logger.info(
+            `remove ressource hint because "${href}" not used anymore`,
+          )
+          actions.push(() => {
+            removeHtmlNode(linkNode)
+          })
+          return
+        }
+        actions.push(() => {
+          setHtmlNodeAttributes(linkNode, {
+            href: urlInfo.data.buildUrlSpecifier,
+          })
+        })
+      }
+      visitHtmlNodes(htmlAst, {
+        link: (node) => {
+          const href = getHtmlNodeAttribute(node, "href")
+          if (href !== undefined) {
+            visitLinkWithHref(node, href)
+          }
+        },
+      })
+      if (actions.length) {
+        actions.forEach((action) => action())
+        await finalGraphKitchen.urlInfoTransformer.applyFinalTransformations(
+          urlInfo,
+          {
+            content: stringifyHtmlAst(htmlAst),
+          },
+        )
+      }
+    })
+  })
+  await Promise.all(
+    ressourceHintActions.map((ressourceHintAction) => ressourceHintAction()),
+  )
+}

package/src/build/start_build_server.js

@@ -0,0 +1,217 @@
+/*
+ * startBuildServer is mean to interact with the build files;
+ * files that will be deployed to production server(s).
+ * We want to be as close as possible from the production in order to:
+ * - run lighthouse
+ * - run an automated test tool such as cypress, playwright
+ * - see exactly how build file behaves (debug, measure perf, etc)
+ * For these reasons "startBuildServer" must be as close as possible from a static file server.
+ * It is not meant to provide a nice developper experience: this is the role "startDevServer".
+ *
+ * Conclusion:
+ * "startBuildServer" must be as close as possible from a static file server because
+ * we want to be in the user shoes and we should not alter build files.
+ */
+
+import {
+  jsenvAccessControlAllowedHeaders,
+  startServer,
+  pluginServerTiming,
+  pluginRequestWaitingCheck,
+  pluginCORS,
+  fetchFileSystem,
+  composeServices,
+  findFreePort,
+} from "@jsenv/server"
+import {
+  assertAndNormalizeDirectoryUrl,
+  registerDirectoryLifecycle,
+} from "@jsenv/filesystem"
+import { Abort, raceProcessTeardownEvents } from "@jsenv/abort"
+import { createLogger, loggerToLevels, createTaskLog } from "@jsenv/log"
+import { getCallerPosition } from "@jsenv/urls"
+
+import { createReloadableWorker } from "@jsenv/core/src/helpers/worker_reload.js"
+
+export const startBuildServer = async ({
+  signal = new AbortController().signal,
+  handleSIGINT = true,
+  logLevel,
+  serverLogLevel = "warn",
+  protocol = "http",
+  http2,
+  certificate,
+  privateKey,
+  listenAnyIp,
+  ip,
+  port = 9779,
+  services = {},
+  keepProcessAlive = true,
+
+  rootDirectoryUrl,
+  buildDirectoryUrl,
+  mainBuildFileUrl = "/index.html",
+  buildServerFiles = {
+    "./package.json": true,
+    "./jsenv.config.mjs": true,
+  },
+  buildServerMainFile = getCallerPosition().url,
+  // force disable server autoreload when this code is executed:
+  // - inside a forked child process
+  // - debugged by vscode
+  // otherwise we get net:ERR_CONNECTION_REFUSED
+  buildServerAutoreload = typeof process.send !== "function" &&
+    !process.env.VSCODE_INSPECTOR_OPTIONS,
+  cooldownBetweenFileEvents,
+}) => {
+  const logger = createLogger({ logLevel })
+  rootDirectoryUrl = assertAndNormalizeDirectoryUrl(rootDirectoryUrl)
+  buildDirectoryUrl = assertAndNormalizeDirectoryUrl(buildDirectoryUrl)
+
+  const operation = Abort.startOperation()
+  operation.addAbortSignal(signal)
+  if (handleSIGINT) {
+    operation.addAbortSource((abort) => {
+      return raceProcessTeardownEvents(
+        {
+          SIGINT: true,
+        },
+        abort,
+      )
+    })
+  }
+  if (port === 0) {
+    port = await findFreePort(port, { signal: operation.signal })
+  }
+
+  const reloadableWorker = createReloadableWorker(buildServerMainFile)
+  if (buildServerAutoreload && reloadableWorker.isPrimary) {
+    const buildServerFileChangeCallback = ({ relativeUrl, event }) => {
+      const url = new URL(relativeUrl, rootDirectoryUrl).href
+      if (buildServerAutoreload) {
+        logger.info(`file ${event} ${url} -> restarting server...`)
+        reloadableWorker.reload()
+      }
+    }
+    const stopWatchingBuildServerFiles = registerDirectoryLifecycle(
+      rootDirectoryUrl,
+      {
+        watchPatterns: {
+          [buildServerMainFile]: true,
+          ...buildServerFiles,
+        },
+        cooldownBetweenFileEvents,
+        keepProcessAlive: false,
+        recursive: true,
+        added: ({ relativeUrl }) => {
+          buildServerFileChangeCallback({ relativeUrl, event: "added" })
+        },
+        updated: ({ relativeUrl }) => {
+          buildServerFileChangeCallback({ relativeUrl, event: "modified" })
+        },
+        removed: ({ relativeUrl }) => {
+          buildServerFileChangeCallback({ relativeUrl, event: "removed" })
+        },
+      },
+    )
+    operation.addAbortCallback(() => {
+      stopWatchingBuildServerFiles()
+      reloadableWorker.terminate()
+    })
+    const worker = await reloadableWorker.load()
+    if (!keepProcessAlive) {
+      worker.unref()
+    }
+    return {
+      origin: `${protocol}://127.0.0.1:${port}`,
+      stop: () => {
+        stopWatchingBuildServerFiles()
+        reloadableWorker.terminate()
+      },
+    }
+  }
+
+  const startBuildServerTask = createTaskLog("start build server", {
+    disabled: !loggerToLevels(logger).info,
+  })
+  const server = await startServer({
+    signal,
+    stopOnExit: false,
+    stopOnSIGINT: false,
+    stopOnInternalError: false,
+    keepProcessAlive: true,
+    logLevel: serverLogLevel,
+    startLog: false,
+
+    protocol,
+    http2,
+    certificate,
+    privateKey,
+    listenAnyIp,
+    ip,
+    port,
+    plugins: {
+      ...pluginCORS({
+        accessControlAllowRequestOrigin: true,
+        accessControlAllowRequestMethod: true,
+        accessControlAllowRequestHeaders: true,
+        accessControlAllowedRequestHeaders: jsenvAccessControlAllowedHeaders,
+        accessControlAllowCredentials: true,
+      }),
+      ...pluginServerTiming(),
+      ...pluginRequestWaitingCheck({
+        requestWaitingMs: 60 * 1000,
+      }),
+    },
+    sendErrorDetails: true,
+    requestToResponse: composeServices({
+      ...services,
+      build_files_service: createBuildFilesService({
+        buildDirectoryUrl,
+        mainBuildFileUrl,
+      }),
+    }),
+  })
+  startBuildServerTask.done()
+  logger.info(``)
+  Object.keys(server.origins).forEach((key) => {
+    logger.info(`- ${server.origins[key]}`)
+  })
+  logger.info(``)
+  return {
+    origin: server.origin,
+    stop: () => {
+      server.stop()
+    },
+  }
+}
+
+const createBuildFilesService = ({ buildDirectoryUrl, mainBuildFileUrl }) => {
+  return (request) => {
+    const urlIsVersioned = new URL(
+      request.ressource,
+      request.origin,
+    ).searchParams.has("v")
+    if (mainBuildFileUrl && request.ressource === "/") {
+      request = {
+        ...request,
+        ressource: mainBuildFileUrl,
+      }
+    }
+    return fetchFileSystem(
+      new URL(request.ressource.slice(1), buildDirectoryUrl),
+      {
+        headers: request.headers,
+        cacheControl: urlIsVersioned
+          ? `private,max-age=${SECONDS_IN_30_DAYS},immutable`
+          : "private,max-age=0,must-revalidate",
+        etagEnabled: true,
+        compressionEnabled: !request.pathname.endsWith(".mp4"),
+        rootDirectoryUrl: buildDirectoryUrl,
+        canReadDirectory: true,
+      },
+    )
+  }
+}
+
+const SECONDS_IN_30_DAYS = 60 * 60 * 24 * 30