@jsenv/core 26.0.1 → 27.0.0-alpha.10

package/src/omega/server/file_service.js

@@ -0,0 +1,155 @@
+import {
+  fetchFileSystem,
+  serveDirectory,
+  composeTwoResponses,
+} from "@jsenv/server"
+import { urlIsInsideOf, moveUrl } from "@jsenv/filesystem"
+
+import { parseUserAgentHeader } from "./user_agent.js"
+
+export const createFileService = ({
+  rootDirectoryUrl,
+  urlGraph,
+  kitchen,
+  scenario,
+}) => {
+  kitchen.pluginController.addHook("serve")
+  kitchen.pluginController.addHook("augmentResponse")
+  const serveContext = {
+    rootDirectoryUrl,
+    urlGraph,
+    scenario,
+  }
+
+  const getResponse = async (request) => {
+    // serve file inside ".jsenv" directory
+    const requestFileUrl = new URL(request.ressource.slice(1), rootDirectoryUrl)
+      .href
+    if (urlIsInsideOf(requestFileUrl, kitchen.jsenvDirectoryUrl)) {
+      return fetchFileSystem(requestFileUrl, {
+        headers: request.headers,
+      })
+    }
+    const responseFromPlugin =
+      await kitchen.pluginController.callAsyncHooksUntil(
+        "serve",
+        request,
+        serveContext,
+      )
+    if (responseFromPlugin) {
+      return responseFromPlugin
+    }
+    const { runtimeName, runtimeVersion } = parseUserAgentHeader(
+      request.headers["user-agent"],
+    )
+    const runtimeSupport = {
+      [runtimeName]: runtimeVersion,
+    }
+    const reference = kitchen.createReference({
+      parentUrl: inferParentFromRequest(request, rootDirectoryUrl),
+      type: "entry_point",
+      specifier: request.ressource,
+    })
+    const requestedUrlInfo = kitchen.resolveReference(reference)
+    const referenceFromGraph = urlGraph.inferReference(
+      reference.url,
+      reference.parentUrl,
+    )
+    try {
+      await kitchen.cook({
+        reference: referenceFromGraph || reference,
+        urlInfo: requestedUrlInfo,
+        outDirectoryUrl: `${rootDirectoryUrl}.jsenv/${scenario}/${runtimeName}@${runtimeVersion}/`,
+        runtimeSupport,
+      })
+      const { response, contentType, content } = requestedUrlInfo
+      if (response) {
+        return response
+      }
+      return {
+        url: reference.url,
+        status: 200,
+        headers: {
+          "content-type": contentType,
+          "content-length": Buffer.byteLength(content),
+          "cache-control": `private,max-age=0,must-revalidate`,
+        },
+        body: content,
+      }
+    } catch (e) {
+      const code = e.code
+      if (code === "PARSE_ERROR") {
+        // let the browser re-throw the syntax error
+        return {
+          url: reference.url,
+          status: 200,
+          statusText: e.reason,
+          statusMessage: e.message,
+          headers: {
+            "content-type": requestedUrlInfo.contentType,
+            "content-length": Buffer.byteLength(requestedUrlInfo.content),
+            "cache-control": "no-store",
+          },
+          body: requestedUrlInfo.content,
+        }
+      }
+      if (code === "EISDIR") {
+        return serveDirectory(reference.url, {
+          headers: {
+            accept: "text/html",
+          },
+          canReadDirectory: true,
+          rootDirectoryUrl,
+        })
+      }
+      if (code === "NOT_ALLOWED") {
+        return {
+          url: reference.url,
+          status: 403,
+          statusText: e.reason,
+        }
+      }
+      if (code === "NOT_FOUND") {
+        return {
+          url: reference.url,
+          status: 404,
+          statusText: e.reason,
+          statusMessage: e.message,
+        }
+      }
+      return {
+        url: reference.url,
+        status: 500,
+        statusText: e.reason,
+        statusMessage: e.stack,
+      }
+    }
+  }
+  return async (request) => {
+    let response = await getResponse(request)
+    if (response.url) {
+      kitchen.pluginController.callHooks(
+        "augmentResponse",
+        response,
+        {},
+        (returnValue) => {
+          response = composeTwoResponses(response, returnValue)
+        },
+      )
+    }
+    return response
+  }
+}
+
+const inferParentFromRequest = (request, rootDirectoryUrl) => {
+  const { referer } = request.headers
+  if (!referer) {
+    return rootDirectoryUrl
+  }
+  return moveUrl({
+    url: referer,
+    from: `${request.origin}/`,
+    to: rootDirectoryUrl,
+    preferAbsolute: true,
+  })
+}

package/src/omega/server/user_agent.js

@@ -0,0 +1,22 @@
+import { createRequire } from "node:module"
+
+const require = createRequire(import.meta.url)
+
+export const parseUserAgentHeader = (userAgent) => {
+  if (userAgent.includes("node-fetch/")) {
+    // it's not really node and conceptually we can't assume the node version
+    // but good enough for now
+    return {
+      runtimeName: "node",
+      runtimeVersion: process.version.slice(1),
+    }
+  }
+  const UA = require("@financial-times/polyfill-useragent-normaliser")
+  const { ua } = new UA(userAgent)
+  const { family, major, minor, patch } = ua
+  return {
+    runtimeName: family.toLowerCase(),
+    runtimeVersion:
+      family === "Other" ? "unknown" : `${major}.${minor}${patch}`,
+  }
+}

package/src/omega/url_graph/url_graph_load.js

@@ -0,0 +1,69 @@
+import { ensureEmptyDirectory } from "@jsenv/filesystem"
+
+export const loadUrlGraph = async ({
+  urlGraph,
+  kitchen,
+  startLoading,
+  outDirectoryUrl,
+  runtimeSupport,
+}) => {
+  if (outDirectoryUrl) {
+    await ensureEmptyDirectory(outDirectoryUrl)
+  }
+  const promises = []
+  const cook = ({ urlInfo, ...rest }) => {
+    const promiseFromData = urlInfo.data.promise
+    if (promiseFromData) return promiseFromData
+    const promise = _cook({
+      urlInfo,
+      outDirectoryUrl,
+      runtimeSupport,
+      ...rest,
+    })
+    promises.push(promise)
+    urlInfo.data.promise = promise
+    return promise
+  }
+  const _cook = async ({ urlInfo, ...rest }) => {
+    await kitchen.cook({
+      urlInfo,
+      cookDuringCook: cook,
+      ...rest,
+    })
+    const { references } = urlInfo
+    references.forEach((reference) => {
+      cook({
+        reference,
+        urlInfo: urlGraph.getUrlInfo(reference.url),
+      })
+    })
+  }
+  startLoading(
+    ({ trace, parentUrl = kitchen.rootDirectoryUrl, type, specifier }) => {
+      const entryReference = kitchen.createReference({
+        trace,
+        parentUrl,
+        type,
+        specifier,
+      })
+      const entryUrlInfo = kitchen.resolveReference(entryReference)
+      entryUrlInfo.data.isEntryPoint = true
+      cook({
+        reference: entryReference,
+        urlInfo: entryUrlInfo,
+      })
+      return [entryReference, entryUrlInfo]
+    },
+  )
+
+  const waitAll = async () => {
+    if (promises.length === 0) {
+      return
+    }
+    const promisesToWait = promises.slice()
+    promises.length = 0
+    await Promise.all(promisesToWait)
+    await waitAll()
+  }
+  await waitAll()
+}

package/src/omega/url_graph/url_graph_report.js

@@ -0,0 +1,151 @@
+import { ANSI } from "@jsenv/log"
+
+import { byteAsFileSize } from "@jsenv/utils/logs/size_log.js"
+
+export const createUrlGraphSummary = (
+  urlGraph,
+  { title = "graph summary" } = {},
+) => {
+  const graphReport = createUrlGraphReport(urlGraph)
+  const totalLabel = `Total`
+  return `--- ${title} ---  
+${createRepartitionMessage(graphReport)}
+${ANSI.color(totalLabel, ANSI.GREY)} ${
+    graphReport.total.count
+  } (${byteAsFileSize(graphReport.total.size)})
+--------------------`
+}
+
+const createUrlGraphReport = (urlGraph) => {
+  const { urlInfos } = urlGraph
+  const countGroups = {
+    html: 0,
+    css: 0,
+    js: 0,
+    assets: 0,
+    sourcemaps: 0,
+    total: 0,
+  }
+  const sizeGroups = {
+    html: 0,
+    css: 0,
+    js: 0,
+    sourcemaps: 0,
+    assets: 0,
+    total: 0,
+  }
+  Object.keys(urlInfos).forEach((url) => {
+    if (url.startsWith("data:")) {
+      return
+    }
+    const urlInfo = urlInfos[url]
+    // ignore inline files, they are already taken into account in the file where they appear
+    if (urlInfo.isInline) {
+      return
+    }
+    // file loaded via import assertion are already inside the graph
+    // their js module equivalent are ignored to avoid counting it twice
+    // in the build graph the file targeted by import assertion will likely be gone
+    // and only the js module remain (likely bundled)
+    const urlObject = new URL(urlInfo.url)
+    if (
+      urlObject.searchParams.has("json_module") ||
+      urlObject.searchParams.has("css_module") ||
+      urlObject.searchParams.has("text_module")
+    ) {
+      return
+    }
+    const urlContentSize = Buffer.byteLength(urlInfo.content)
+    const category = determineCategory(urlInfo)
+    if (category === "sourcemap") {
+      countGroups.sourcemaps++
+      sizeGroups.sourcemaps += urlContentSize
+      return
+    }
+    countGroups.total++
+    sizeGroups.total += urlContentSize
+    if (category === "html") {
+      countGroups.html++
+      sizeGroups.html += urlContentSize
+      return
+    }
+    if (category === "css") {
+      countGroups.css++
+      sizeGroups.css += urlContentSize
+      return
+    }
+    if (category === "js") {
+      countGroups.js++
+      sizeGroups.js += urlContentSize
+      return
+    }
+    countGroups.assets++
+    sizeGroups.assets += urlContentSize
+    return
+  })
+  return {
+    html: { count: countGroups.html, size: sizeGroups.html },
+    css: { count: countGroups.css, size: sizeGroups.css },
+    js: { count: countGroups.js, size: sizeGroups.js },
+    sourcemaps: { count: countGroups.sourcemaps, size: sizeGroups.sourcemaps },
+    assets: { count: countGroups.assets, size: sizeGroups.assets },
+    total: { count: countGroups.total, size: sizeGroups.total },
+  }
+}
+
+const determineCategory = (urlInfo) => {
+  if (urlInfo.type === "sourcemap") {
+    return "sourcemap"
+  }
+  if (urlInfo.type === "html") {
+    return "html"
+  }
+  if (urlInfo.type === "css") {
+    return "css"
+  }
+  if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
+    return "js"
+  }
+  return "assets"
+}
+
+const createRepartitionMessage = ({ html, css, js, assets }) => {
+  const parts = []
+  if (html.count) {
+    parts.push(
+      `${ANSI.color(`html:`, ANSI.GREY)} ${html.count} (${byteAsFileSize(
+        html.size,
+      )})`,
+    )
+  }
+  if (css.count) {
+    parts.push(
+      `${ANSI.color(`css:`, ANSI.GREY)} ${css.count} (${byteAsFileSize(
+        css.size,
+      )})`,
+    )
+  }
+  if (js.count) {
+    parts.push(
+      `${ANSI.color(`js:`, ANSI.GREY)} ${js.count} (${byteAsFileSize(
+        js.size,
+      )})`,
+    )
+  }
+  // if (sourcemaps.count) {
+  //   parts.push(
+  //     `${ANSI.color(`sourcemaps:`, ANSI.GREY)} ${
+  //       sourcemaps.count
+  //     } (${byteAsFileSize(sourcemaps.size)})`,
+  //   )
+  // }
+  if (assets.count) {
+    parts.push(
+      `${ANSI.color(`assets:`, ANSI.GREY)} ${assets.count} (${byteAsFileSize(
+        assets.size,
+      )})`,
+    )
+  }
+  return `- ${parts.join(`
+- `)}`
+}

package/src/omega/url_graph/url_graph_sort.js

@@ -0,0 +1,29 @@
+export const sortUrlGraphByDependencies = (urlGraph) => {
+  const { urlInfos } = urlGraph
+
+  const visited = []
+  const sorted = []
+  const circular = []
+  const visit = (url) => {
+    const isSorted = sorted.includes(url)
+    if (isSorted) {
+      return
+    }
+    const isVisited = visited.includes(url)
+    if (isVisited) {
+      circular.push(url)
+      sorted.push(url)
+    } else {
+      visited.push(url)
+      urlInfos[url].dependencies.forEach((dependencyUrl) => {
+        visit(dependencyUrl, url)
+      })
+      sorted.push(url)
+    }
+  }
+  Object.keys(urlInfos).forEach((url) => {
+    visit(url)
+  })
+  sorted.circular = circular
+  return sorted
+}

package/src/omega/url_graph/url_info_transformations.js

@@ -0,0 +1,161 @@
+import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+import {
+  SOURCEMAP,
+  sourcemapToBase64Url,
+  generateSourcemapUrl,
+} from "@jsenv/utils/sourcemap/sourcemap_utils.js"
+
+export const createUrlInfoTransformer = ({
+  logger,
+  sourcemaps,
+  sourcemapsSources,
+  urlGraph,
+  injectSourcemapPlaceholder,
+  foundSourcemap,
+}) => {
+  const sourcemapsEnabled =
+    sourcemaps === "inline" ||
+    sourcemaps === "file" ||
+    sourcemaps === "programmatic"
+
+  const normalizeSourcemap = (urlInfo, sourcemap) => {
+    const wantSourcesContent =
+      // for inline content (<script> insdide html)
+      // chrome won't be able to fetch the file as it does not exists
+      // so sourcemap must contain sources
+      urlInfo.isInline || sourcemapsSources
+    if (sourcemap.sources && sourcemap.sources.length > 1) {
+      sourcemap.sources = sourcemap.sources.map(
+        (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
+      )
+      if (!wantSourcesContent) {
+        sourcemap.sourcesContent = undefined
+      }
+      return sourcemap
+    }
+    sourcemap.sources = [urlInfo.data.rawUrl || urlInfo.url]
+    sourcemap.sourcesContent = [urlInfo.originalContent]
+    if (!wantSourcesContent) {
+      sourcemap.sourcesContent = undefined
+    }
+    return sourcemap
+  }
+
+  const initTransformations = async (urlInfo, context) => {
+    if (!sourcemapsEnabled) {
+      return
+    }
+    if (!SOURCEMAP.enabledOnContentType(urlInfo.contentType)) {
+      return
+    }
+    // sourcemap is a special kind of reference:
+    // It's a reference to a content generated dynamically the content itself.
+    // For this reason sourcemap are not added to urlInfo.references
+    // Instead they are stored into urlInfo.sourcemapReference
+    // create a placeholder reference for the sourcemap that will be generated
+    // when jsenv is done cooking the file
+    //   during build it's urlInfo.url to be inside the build
+    //   but otherwise it's generatedUrl to be inside .jsenv/ directory
+    urlInfo.sourcemapGeneratedUrl = generateSourcemapUrl(urlInfo.generatedUrl)
+    const [sourcemapReference, sourcemapUrlInfo] = injectSourcemapPlaceholder({
+      urlInfo,
+      specifier: urlInfo.sourcemapGeneratedUrl,
+    })
+    urlInfo.sourcemapReference = sourcemapReference
+    sourcemapUrlInfo.isInline = sourcemaps === "inline"
+
+    // already loaded during "load" hook (happens during build)
+    if (urlInfo.sourcemap) {
+      return
+    }
+    // check for existing sourcemap for this content
+    const sourcemapFound = SOURCEMAP.readComment({
+      contentType: urlInfo.contentType,
+      content: urlInfo.content,
+    })
+    if (sourcemapFound) {
+      const { type, line, column, specifier } = sourcemapFound
+      const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
+        urlInfo,
+        type,
+        line,
+        column,
+        specifier,
+      })
+      try {
+        await context.cook({
+          reference: sourcemapReference,
+          urlInfo: sourcemapUrlInfo,
+        })
+        const sourcemap = JSON.parse(sourcemapUrlInfo.content)
+        urlInfo.sourcemap = normalizeSourcemap(urlInfo, sourcemap)
+      } catch (e) {
+        logger.error(`Error while handling existing sourcemap: ${e.message}`)
+        return
+      }
+    }
+  }
+
+  const applyIntermediateTransformations = async (urlInfo, transformations) => {
+    if (!transformations) {
+      return
+    }
+    const { type, contentType, content, sourcemap } = transformations
+    if (type) {
+      urlInfo.type = type
+    }
+    if (contentType) {
+      urlInfo.contentType = contentType
+    }
+    if (content) {
+      urlInfo.content = content
+    }
+    if (sourcemapsEnabled && sourcemap) {
+      const sourcemapNormalized = normalizeSourcemap(urlInfo, sourcemap)
+      const finalSourcemap = await composeTwoSourcemaps(
+        urlInfo.sourcemap,
+        sourcemapNormalized,
+      )
+      const finalSourcemapNormalized = normalizeSourcemap(
+        urlInfo,
+        finalSourcemap,
+      )
+      urlInfo.sourcemap = finalSourcemapNormalized
+    }
+  }
+
+  const applyFinalTransformations = async (urlInfo, transformations) => {
+    if (transformations) {
+      await applyIntermediateTransformations(urlInfo, transformations)
+    }
+    if (sourcemapsEnabled && urlInfo.sourcemap) {
+      // during build this function can be called after the file is cooked
+      // - to update content and sourcemap after "optimize" hook
+      // - to inject versioning into the entry point content
+      // in this scenarion we don't want to call injectSourcemap
+      // just update the content and the
+      const sourcemapReference = urlInfo.sourcemapReference
+      const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
+      sourcemapUrlInfo.contentType = "application/json"
+      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      if (sourcemaps === "inline") {
+        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
+          urlInfo.sourcemap,
+        )
+      }
+      if (sourcemaps === "file" || sourcemaps === "inline") {
+        urlInfo.content = SOURCEMAP.writeComment({
+          contentType: urlInfo.contentType,
+          content: urlInfo.content,
+          specifier: sourcemapReference.generatedSpecifier,
+        })
+      }
+    }
+  }
+
+  return {
+    initTransformations,
+    applyIntermediateTransformations,
+    applyFinalTransformations,
+  }
+}