@jsenv/core 26.0.1 → 27.0.0-alpha.2

package/src/omega/url_graph/url_graph_load.js

@@ -0,0 +1,68 @@
+import { ensureEmptyDirectory } from "@jsenv/filesystem"
+
+export const loadUrlGraph = async ({
+  urlGraph,
+  kitchen,
+  startLoading,
+  outDirectoryUrl,
+  runtimeSupport,
+}) => {
+  if (outDirectoryUrl) {
+    await ensureEmptyDirectory(outDirectoryUrl)
+  }
+  const promises = []
+  const cook = ({ urlInfo, ...rest }) => {
+    const promiseFromData = urlInfo.data.promise
+    if (promiseFromData) return promiseFromData
+    const promise = _cook({
+      urlInfo,
+      outDirectoryUrl,
+      runtimeSupport,
+      ...rest,
+    })
+    promises.push(promise)
+    urlInfo.data.promise = promise
+    return promise
+  }
+  const _cook = async ({ urlInfo, ...rest }) => {
+    await kitchen.cook({
+      urlInfo,
+      cookDuringCook: cook,
+      ...rest,
+    })
+    const { references } = urlInfo
+    references.forEach((reference) => {
+      cook({
+        reference,
+        urlInfo: urlGraph.getUrlInfo(reference.url),
+      })
+    })
+  }
+  startLoading(
+    ({ trace, parentUrl = kitchen.rootDirectoryUrl, type, specifier }) => {
+      const entryReference = kitchen.createReference({
+        trace,
+        parentUrl,
+        type,
+        specifier,
+      })
+      const entryUrlInfo = kitchen.resolveReference(entryReference)
+      entryUrlInfo.data.isEntryPoint = true
+      cook({
+        reference: entryReference,
+        urlInfo: entryUrlInfo,
+      })
+    },
+  )
+
+  const waitAll = async () => {
+    if (promises.length === 0) {
+      return
+    }
+    const promisesToWait = promises.slice()
+    promises.length = 0
+    await Promise.all(promisesToWait)
+    await waitAll()
+  }
+  await waitAll()
+}

package/src/omega/url_graph/url_graph_report.js

@@ -0,0 +1,151 @@
+import { ANSI } from "@jsenv/log"
+
+import { byteAsFileSize } from "@jsenv/utils/logs/size_log.js"
+
+export const createUrlGraphSummary = (
+  urlGraph,
+  { title = "graph summary" } = {},
+) => {
+  const graphReport = createUrlGraphReport(urlGraph)
+  const totalLabel = `Total`
+  return `--- ${title} ---  
+${createRepartitionMessage(graphReport)}
+${ANSI.color(totalLabel, ANSI.GREY)} ${
+    graphReport.total.count
+  } (${byteAsFileSize(graphReport.total.size)})
+--------------------`
+}
+
+const createUrlGraphReport = (urlGraph) => {
+  const { urlInfos } = urlGraph
+  const countGroups = {
+    html: 0,
+    css: 0,
+    js: 0,
+    assets: 0,
+    sourcemaps: 0,
+    total: 0,
+  }
+  const sizeGroups = {
+    html: 0,
+    css: 0,
+    js: 0,
+    sourcemaps: 0,
+    assets: 0,
+    total: 0,
+  }
+  Object.keys(urlInfos).forEach((url) => {
+    if (url.startsWith("data:")) {
+      return
+    }
+    const urlInfo = urlInfos[url]
+    // ignore inline files, they are already taken into account in the file where they appear
+    if (urlInfo.isInline) {
+      return
+    }
+    // file loaded via import assertion are already inside the graph
+    // their js module equivalent are ignored to avoid counting it twice
+    // in the build graph the file targeted by import assertion will likely be gone
+    // and only the js module remain (likely bundled)
+    const urlObject = new URL(urlInfo.url)
+    if (
+      urlObject.searchParams.has("json_module") ||
+      urlObject.searchParams.has("css_module") ||
+      urlObject.searchParams.has("text_module")
+    ) {
+      return
+    }
+    const urlContentSize = Buffer.byteLength(urlInfo.content)
+    const category = determineCategory(urlInfo)
+    if (category === "sourcemap") {
+      countGroups.sourcemaps++
+      sizeGroups.sourcemaps += urlContentSize
+      return
+    }
+    countGroups.total++
+    sizeGroups.total += urlContentSize
+    if (category === "html") {
+      countGroups.html++
+      sizeGroups.html += urlContentSize
+      return
+    }
+    if (category === "css") {
+      countGroups.css++
+      sizeGroups.css += urlContentSize
+      return
+    }
+    if (category === "js") {
+      countGroups.js++
+      sizeGroups.js += urlContentSize
+      return
+    }
+    countGroups.assets++
+    sizeGroups.assets += urlContentSize
+    return
+  })
+  return {
+    html: { count: countGroups.html, size: sizeGroups.html },
+    css: { count: countGroups.css, size: sizeGroups.css },
+    js: { count: countGroups.js, size: sizeGroups.js },
+    sourcemaps: { count: countGroups.sourcemaps, size: sizeGroups.sourcemaps },
+    assets: { count: countGroups.assets, size: sizeGroups.assets },
+    total: { count: countGroups.total, size: sizeGroups.total },
+  }
+}
+
+const determineCategory = (urlInfo) => {
+  if (urlInfo.type === "sourcemap") {
+    return "sourcemap"
+  }
+  if (urlInfo.type === "html") {
+    return "html"
+  }
+  if (urlInfo.type === "css") {
+    return "css"
+  }
+  if (urlInfo.type === "js_module" || urlInfo.type === "js_classic") {
+    return "js"
+  }
+  return "assets"
+}
+
+const createRepartitionMessage = ({ html, css, js, assets }) => {
+  const parts = []
+  if (html.count) {
+    parts.push(
+      `${ANSI.color(`html:`, ANSI.GREY)} ${html.count} (${byteAsFileSize(
+        html.size,
+      )})`,
+    )
+  }
+  if (css.count) {
+    parts.push(
+      `${ANSI.color(`css:`, ANSI.GREY)} ${css.count} (${byteAsFileSize(
+        css.size,
+      )})`,
+    )
+  }
+  if (js.count) {
+    parts.push(
+      `${ANSI.color(`js:`, ANSI.GREY)} ${js.count} (${byteAsFileSize(
+        js.size,
+      )})`,
+    )
+  }
+  // if (sourcemaps.count) {
+  //   parts.push(
+  //     `${ANSI.color(`sourcemaps:`, ANSI.GREY)} ${
+  //       sourcemaps.count
+  //     } (${byteAsFileSize(sourcemaps.size)})`,
+  //   )
+  // }
+  if (assets.count) {
+    parts.push(
+      `${ANSI.color(`assets:`, ANSI.GREY)} ${assets.count} (${byteAsFileSize(
+        assets.size,
+      )})`,
+    )
+  }
+  return `- ${parts.join(`
+- `)}`
+}

package/src/omega/url_graph/url_graph_sort.js

@@ -0,0 +1,29 @@
+export const sortUrlGraphByDependencies = (urlGraph) => {
+  const { urlInfos } = urlGraph
+
+  const visited = []
+  const sorted = []
+  const circular = []
+  const visit = (url) => {
+    const isSorted = sorted.includes(url)
+    if (isSorted) {
+      return
+    }
+    const isVisited = visited.includes(url)
+    if (isVisited) {
+      circular.push(url)
+      sorted.push(url)
+    } else {
+      visited.push(url)
+      urlInfos[url].dependencies.forEach((dependencyUrl) => {
+        visit(dependencyUrl, url)
+      })
+      sorted.push(url)
+    }
+  }
+  Object.keys(urlInfos).forEach((url) => {
+    visit(url)
+  })
+  sorted.circular = circular
+  return sorted
+}

package/src/omega/url_graph/url_info_transformations.js

@@ -0,0 +1,161 @@
+import { composeTwoSourcemaps } from "@jsenv/utils/sourcemap/sourcemap_composition_v3.js"
+import {
+  SOURCEMAP,
+  sourcemapToBase64Url,
+  generateSourcemapUrl,
+} from "@jsenv/utils/sourcemap/sourcemap_utils.js"
+
+export const createUrlInfoTransformer = ({
+  logger,
+  sourcemaps,
+  sourcemapsSources,
+  urlGraph,
+  injectSourcemapPlaceholder,
+  foundSourcemap,
+}) => {
+  const sourcemapsEnabled =
+    sourcemaps === "inline" ||
+    sourcemaps === "file" ||
+    sourcemaps === "programmatic"
+
+  const normalizeSourcemap = (urlInfo, sourcemap) => {
+    const wantSourcesContent =
+      // for inline content (<script> insdide html)
+      // chrome won't be able to fetch the file as it does not exists
+      // so sourcemap must contain sources
+      urlInfo.isInline || sourcemapsSources
+    if (sourcemap.sources && sourcemap.sources.length > 1) {
+      sourcemap.sources = sourcemap.sources.map(
+        (source) => new URL(source, urlInfo.data.rawUrl || urlInfo.url).href,
+      )
+      if (!wantSourcesContent) {
+        sourcemap.sourcesContent = undefined
+      }
+      return sourcemap
+    }
+    sourcemap.sources = [urlInfo.data.rawUrl || urlInfo.url]
+    sourcemap.sourcesContent = [urlInfo.originalContent]
+    if (!wantSourcesContent) {
+      sourcemap.sourcesContent = undefined
+    }
+    return sourcemap
+  }
+
+  const initTransformations = async (urlInfo, context) => {
+    if (!sourcemapsEnabled) {
+      return
+    }
+    if (!SOURCEMAP.enabledOnContentType(urlInfo.contentType)) {
+      return
+    }
+    // sourcemap is a special kind of reference:
+    // It's a reference to a content generated dynamically the content itself.
+    // For this reason sourcemap are not added to urlInfo.references
+    // Instead they are stored into urlInfo.sourcemapReference
+    // create a placeholder reference for the sourcemap that will be generated
+    // when jsenv is done cooking the file
+    //   during build it's urlInfo.url to be inside the build
+    //   but otherwise it's generatedUrl to be inside .jsenv/ directory
+    urlInfo.sourcemapGeneratedUrl = generateSourcemapUrl(urlInfo.generatedUrl)
+    const [sourcemapReference, sourcemapUrlInfo] = injectSourcemapPlaceholder({
+      urlInfo,
+      specifier: urlInfo.sourcemapGeneratedUrl,
+    })
+    urlInfo.sourcemapReference = sourcemapReference
+    sourcemapUrlInfo.isInline = sourcemaps === "inline"
+
+    // already loaded during "load" hook (happens during build)
+    if (urlInfo.sourcemap) {
+      return
+    }
+    // check for existing sourcemap for this content
+    const sourcemapFound = SOURCEMAP.readComment({
+      contentType: urlInfo.contentType,
+      content: urlInfo.content,
+    })
+    if (sourcemapFound) {
+      const { type, line, column, specifier } = sourcemapFound
+      const [sourcemapReference, sourcemapUrlInfo] = foundSourcemap({
+        urlInfo,
+        type,
+        line,
+        column,
+        specifier,
+      })
+      try {
+        await context.cook({
+          reference: sourcemapReference,
+          urlInfo: sourcemapUrlInfo,
+        })
+        const sourcemap = JSON.parse(sourcemapUrlInfo.content)
+        urlInfo.sourcemap = normalizeSourcemap(urlInfo, sourcemap)
+      } catch (e) {
+        logger.error(`Error while handling existing sourcemap: ${e.message}`)
+        return
+      }
+    }
+  }
+
+  const applyIntermediateTransformations = async (urlInfo, transformations) => {
+    if (!transformations) {
+      return
+    }
+    const { type, contentType, content, sourcemap } = transformations
+    if (type) {
+      urlInfo.type = type
+    }
+    if (contentType) {
+      urlInfo.contentType = contentType
+    }
+    if (content) {
+      urlInfo.content = content
+    }
+    if (sourcemapsEnabled && sourcemap) {
+      const sourcemapNormalized = normalizeSourcemap(urlInfo, sourcemap)
+      const finalSourcemap = await composeTwoSourcemaps(
+        urlInfo.sourcemap,
+        sourcemapNormalized,
+      )
+      const finalSourcemapNormalized = normalizeSourcemap(
+        urlInfo,
+        finalSourcemap,
+      )
+      urlInfo.sourcemap = finalSourcemapNormalized
+    }
+  }
+
+  const applyFinalTransformations = async (urlInfo, transformations) => {
+    if (transformations) {
+      await applyIntermediateTransformations(urlInfo, transformations)
+    }
+    if (sourcemapsEnabled && urlInfo.sourcemap) {
+      // during build this function can be called after the file is cooked
+      // - to update content and sourcemap after "optimize" hook
+      // - to inject versioning into the entry point content
+      // in this scenarion we don't want to call injectSourcemap
+      // just update the content and the
+      const sourcemapReference = urlInfo.sourcemapReference
+      const sourcemapUrlInfo = urlGraph.getUrlInfo(sourcemapReference.url)
+      sourcemapUrlInfo.contentType = "application/json"
+      sourcemapUrlInfo.content = JSON.stringify(urlInfo.sourcemap, null, "  ")
+      if (sourcemaps === "inline") {
+        sourcemapReference.generatedSpecifier = sourcemapToBase64Url(
+          urlInfo.sourcemap,
+        )
+      }
+      if (sourcemaps === "file" || sourcemaps === "inline") {
+        urlInfo.content = SOURCEMAP.writeComment({
+          contentType: urlInfo.contentType,
+          content: urlInfo.content,
+          specifier: sourcemapReference.generatedSpecifier,
+        })
+      }
+    }
+  }
+
+  return {
+    initTransformations,
+    applyIntermediateTransformations,
+    applyFinalTransformations,
+  }
+}

package/src/omega/url_graph.js

@@ -0,0 +1,132 @@
+import { createCallbackList } from "@jsenv/abort"
+import { urlToRelativeUrl } from "@jsenv/filesystem"
+
+export const createUrlGraph = () => {
+  const urlInfos = {}
+  const getUrlInfo = (url) => urlInfos[url]
+  const deleteUrlInfo = (url) => delete urlInfos[url]
+  const reuseOrCreateUrlInfo = (url) => {
+    const existingUrlInfo = urlInfos[url]
+    if (existingUrlInfo) return existingUrlInfo
+    const urlInfo = createUrlInfo(url)
+    urlInfos[url] = urlInfo
+    return urlInfo
+  }
+  const inferReference = (url, parentUrl) => {
+    const parentUrlInfo = urlInfos[parentUrl]
+    if (!parentUrlInfo) {
+      return null
+    }
+    const firstReferenceOnThatUrl = parentUrlInfo.references.find(
+      (reference) => reference.url === url,
+    )
+    return firstReferenceOnThatUrl
+  }
+  const findDependent = (url, predicate) => {
+    const urlInfo = urlInfos[url]
+    if (!urlInfo) {
+      return null
+    }
+    const visitDependents = (urlInfo) => {
+      for (const dependentUrl of urlInfo.dependents) {
+        const dependent = urlInfos[dependentUrl]
+        if (predicate(dependent)) {
+          return dependent
+        }
+        return visitDependents(dependent)
+      }
+      return null
+    }
+    return visitDependents(urlInfo)
+  }
+
+  const prunedCallbackList = createCallbackList()
+  const updateReferences = (urlInfo, references) => {
+    const dependencyUrls = []
+    references.forEach((reference) => {
+      if (!dependencyUrls.includes(reference.url)) {
+        dependencyUrls.push(reference.url)
+      }
+    })
+    pruneDependencies(
+      urlInfo,
+      Array.from(urlInfo.dependencies).filter(
+        (dep) => !dependencyUrls.includes(dep),
+      ),
+    )
+    urlInfo.references = references
+    dependencyUrls.forEach((dependencyUrl) => {
+      const dependencyUrlInfo = reuseOrCreateUrlInfo(dependencyUrl)
+      urlInfo.dependencies.add(dependencyUrl)
+      dependencyUrlInfo.dependents.add(urlInfo.url)
+    })
+    return urlInfo
+  }
+  const pruneDependencies = (firstUrlInfo, urlsToRemove) => {
+    const prunedUrlInfos = []
+    const removeDependencies = (urlInfo, urlsToPrune) => {
+      urlsToPrune.forEach((urlToPrune) => {
+        urlInfo.dependencies.delete(urlToPrune)
+        const dependency = urlInfos[urlToPrune]
+        if (!dependency) {
+          return
+        }
+        dependency.dependents.delete(urlInfo.url)
+        if (dependency.dependents.size === 0) {
+          removeDependencies(dependency, Array.from(dependency.dependencies))
+          prunedUrlInfos.push(dependency)
+        }
+      })
+    }
+    removeDependencies(firstUrlInfo, urlsToRemove)
+    if (prunedUrlInfos.length === 0) {
+      return
+    }
+    prunedCallbackList.notify({ prunedUrlInfos, firstUrlInfo })
+  }
+
+  return {
+    urlInfos,
+    reuseOrCreateUrlInfo,
+    getUrlInfo,
+    deleteUrlInfo,
+    inferReference,
+    findDependent,
+
+    prunedCallbackList,
+    updateReferences,
+
+    toJSON: (rootDirectoryUrl) => {
+      const data = {}
+      Object.keys(urlInfos).forEach((url) => {
+        const dependencyUrls = Array.from(urlInfos[url].dependencies)
+        if (dependencyUrls.length) {
+          const relativeUrl = urlToRelativeUrl(url, rootDirectoryUrl)
+          data[relativeUrl] = dependencyUrls.map((dependencyUrl) =>
+            urlToRelativeUrl(dependencyUrl, rootDirectoryUrl),
+          )
+        }
+      })
+      return data
+    },
+  }
+}
+
+const createUrlInfo = (url) => {
+  return {
+    data: {}, // plugins can put whatever they want here
+    url,
+    generatedUrl: null,
+    isInline: false,
+    inlineUrlSite: null,
+    contentType: "",
+    originalContent: "",
+    content: "",
+    sourcemap: null,
+    sourcemapReference: null,
+    type: "",
+    references: [],
+    dependencies: new Set(),
+    dependents: new Set(),
+  }
+}

package/src/omega/url_mentions/css_url_mentions.js

@@ -0,0 +1,63 @@
+/*
+ * https://github.com/parcel-bundler/parcel/blob/v2/packages/transformers/css/src/CSSTransformer.js
+ */
+
+import { applyPostCss } from "@jsenv/utils/css_ast/apply_post_css.js"
+import { postCssPluginUrlVisitor } from "@jsenv/utils/css_ast/postcss_plugin_url_visitor.js"
+import { replaceCssUrls } from "@jsenv/utils/css_ast/replace_css_urls.js"
+
+export const parseCssUrlMentions = async ({ url, content }) => {
+  const cssUrlMentions = []
+  await applyPostCss({
+    sourcemaps: false,
+    plugins: [
+      postCssPluginUrlVisitor({
+        urlVisitor: ({ type, url, specifier, urlNode }) => {
+          cssUrlMentions.push({
+            type: `css_${type}`,
+            specifier,
+            start: urlNode.sourceIndex,
+            end: urlNode.sourceEndIndex,
+            url,
+          })
+        },
+      }),
+    ],
+    url,
+    content,
+  })
+  return {
+    urlMentions: cssUrlMentions,
+    replaceUrls: async (getReplacement) => {
+      // we can't use magic source because urlMention.start/end do not match the url specifier
+      // const magicSource = createMagicSource({ url, content })
+      // cssUrlMentions.forEach((urlMention) => {
+      //   magicSource.replace({
+      //     start: urlMention.start,
+      //     end: urlMention.end,
+      //     replacement: JSON.stringify(transformUrlMention(urlMention)),
+      //   })
+      // })
+      // return magicSource.toContentAndSourcemap()
+      const result = await replaceCssUrls({
+        url,
+        content,
+        urlVisitor: ({ url, replace }) => {
+          const cssUrlMention = cssUrlMentions.find(
+            (urlMention) => urlMention.url === url,
+          )
+          if (cssUrlMention) {
+            const replacement = getReplacement(cssUrlMention)
+            if (replacement) {
+              replace(replacement)
+            }
+          }
+        },
+      })
+      return {
+        content: result.content,
+        sourcemap: result.map,
+      }
+    },
+  }
+}