@jsenv/core 26.0.1 → 27.0.0-alpha.10

package/src/omega/url_graph.js

@@ -0,0 +1,133 @@
+import { createCallbackList } from "@jsenv/abort"
+import { urlToRelativeUrl } from "@jsenv/filesystem"
+
+export const createUrlGraph = () => {
+  const urlInfos = {}
+  const getUrlInfo = (url) => urlInfos[url]
+  const deleteUrlInfo = (url) => delete urlInfos[url]
+  const reuseOrCreateUrlInfo = (url) => {
+    const existingUrlInfo = urlInfos[url]
+    if (existingUrlInfo) return existingUrlInfo
+    const urlInfo = createUrlInfo(url)
+    urlInfos[url] = urlInfo
+    return urlInfo
+  }
+  const inferReference = (url, parentUrl) => {
+    const parentUrlInfo = urlInfos[parentUrl]
+    if (!parentUrlInfo) {
+      return null
+    }
+    const firstReferenceOnThatUrl = parentUrlInfo.references.find(
+      (reference) => reference.url === url,
+    )
+    return firstReferenceOnThatUrl
+  }
+  const findDependent = (url, predicate) => {
+    const urlInfo = urlInfos[url]
+    if (!urlInfo) {
+      return null
+    }
+    const visitDependents = (urlInfo) => {
+      for (const dependentUrl of urlInfo.dependents) {
+        const dependent = urlInfos[dependentUrl]
+        if (predicate(dependent)) {
+          return dependent
+        }
+        return visitDependents(dependent)
+      }
+      return null
+    }
+    return visitDependents(urlInfo)
+  }
+
+  const prunedCallbackList = createCallbackList()
+  const updateReferences = (urlInfo, references) => {
+    const dependencyUrls = []
+    references.forEach((reference) => {
+      if (!dependencyUrls.includes(reference.url)) {
+        dependencyUrls.push(reference.url)
+      }
+    })
+    pruneDependencies(
+      urlInfo,
+      Array.from(urlInfo.dependencies).filter(
+        (dep) => !dependencyUrls.includes(dep),
+      ),
+    )
+    urlInfo.references = references
+    dependencyUrls.forEach((dependencyUrl) => {
+      const dependencyUrlInfo = reuseOrCreateUrlInfo(dependencyUrl)
+      urlInfo.dependencies.add(dependencyUrl)
+      dependencyUrlInfo.dependents.add(urlInfo.url)
+    })
+    return urlInfo
+  }
+  const pruneDependencies = (firstUrlInfo, urlsToRemove) => {
+    const prunedUrlInfos = []
+    const removeDependencies = (urlInfo, urlsToPrune) => {
+      urlsToPrune.forEach((urlToPrune) => {
+        urlInfo.dependencies.delete(urlToPrune)
+        const dependency = urlInfos[urlToPrune]
+        if (!dependency) {
+          return
+        }
+        dependency.dependents.delete(urlInfo.url)
+        if (dependency.dependents.size === 0) {
+          removeDependencies(dependency, Array.from(dependency.dependencies))
+          prunedUrlInfos.push(dependency)
+        }
+      })
+    }
+    removeDependencies(firstUrlInfo, urlsToRemove)
+    if (prunedUrlInfos.length === 0) {
+      return
+    }
+    prunedCallbackList.notify({ prunedUrlInfos, firstUrlInfo })
+  }
+
+  return {
+    urlInfos,
+    reuseOrCreateUrlInfo,
+    getUrlInfo,
+    deleteUrlInfo,
+    inferReference,
+    findDependent,
+
+    prunedCallbackList,
+    updateReferences,
+
+    toJSON: (rootDirectoryUrl) => {
+      const data = {}
+      Object.keys(urlInfos).forEach((url) => {
+        const dependencyUrls = Array.from(urlInfos[url].dependencies)
+        if (dependencyUrls.length) {
+          const relativeUrl = urlToRelativeUrl(url, rootDirectoryUrl)
+          data[relativeUrl] = dependencyUrls.map((dependencyUrl) =>
+            urlToRelativeUrl(dependencyUrl, rootDirectoryUrl),
+          )
+        }
+      })
+      return data
+    },
+  }
+}
+
+const createUrlInfo = (url) => {
+  return {
+    data: {}, // plugins can put whatever they want here
+    url,
+    generatedUrl: null,
+    isInline: false,
+    inlineUrlSite: null,
+    contentType: "",
+    originalContent: "",
+    content: "",
+    sourcemap: null,
+    sourcemapReference: null,
+    type: "",
+    subtype: "",
+    references: [],
+    dependencies: new Set(),
+    dependents: new Set(),
+  }
+}

package/src/omega/url_mentions/css_url_mentions.js

@@ -0,0 +1,63 @@
+/*
+ * https://github.com/parcel-bundler/parcel/blob/v2/packages/transformers/css/src/CSSTransformer.js
+ */
+
+import { applyPostCss } from "@jsenv/utils/css_ast/apply_post_css.js"
+import { postCssPluginUrlVisitor } from "@jsenv/utils/css_ast/postcss_plugin_url_visitor.js"
+import { replaceCssUrls } from "@jsenv/utils/css_ast/replace_css_urls.js"
+
+export const parseCssUrlMentions = async ({ url, content }) => {
+  const cssUrlMentions = []
+  await applyPostCss({
+    sourcemaps: false,
+    plugins: [
+      postCssPluginUrlVisitor({
+        urlVisitor: ({ type, url, specifier, urlNode }) => {
+          cssUrlMentions.push({
+            type: `css_${type}`,
+            specifier,
+            start: urlNode.sourceIndex,
+            end: urlNode.sourceEndIndex,
+            url,
+          })
+        },
+      }),
+    ],
+    url,
+    content,
+  })
+  return {
+    urlMentions: cssUrlMentions,
+    replaceUrls: async (getReplacement) => {
+      // we can't use magic source because urlMention.start/end do not match the url specifier
+      // const magicSource = createMagicSource({ url, content })
+      // cssUrlMentions.forEach((urlMention) => {
+      //   magicSource.replace({
+      //     start: urlMention.start,
+      //     end: urlMention.end,
+      //     replacement: JSON.stringify(transformUrlMention(urlMention)),
+      //   })
+      // })
+      // return magicSource.toContentAndSourcemap()
+      const result = await replaceCssUrls({
+        url,
+        content,
+        urlVisitor: ({ url, replace }) => {
+          const cssUrlMention = cssUrlMentions.find(
+            (urlMention) => urlMention.url === url,
+          )
+          if (cssUrlMention) {
+            const replacement = getReplacement(cssUrlMention)
+            if (replacement) {
+              replace(replacement)
+            }
+          }
+        },
+      })
+      return {
+        content: result.content,
+        sourcemap: result.map,
+      }
+    },
+  }
+}

package/src/omega/url_mentions/html_url_mentions.js

@@ -0,0 +1,185 @@
+import {
+  parseHtmlString,
+  stringifyHtmlAst,
+  getHtmlNodeAttributeByName,
+  htmlNodePosition,
+  visitHtmlAst,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import { htmlAttributeSrcSet } from "@jsenv/utils/html_ast/html_attribute_src_set.js"
+
+export const parseHtmlUrlMentions = ({ url, content, scenario }) => {
+  const htmlAst = parseHtmlString(content, {
+    storeOriginalPositions: scenario !== "build",
+  })
+  const htmlUrlMentions = collectHtmlUrlMentions({ url, htmlAst })
+  return {
+    urlMentions: htmlUrlMentions,
+    replaceUrls: (getReplacement) => {
+      htmlUrlMentions.forEach((urlMention) => {
+        const replacement = getReplacement(urlMention)
+        if (replacement) {
+          urlMention.attribute.value = replacement
+        }
+      })
+      return {
+        content: stringifyHtmlAst(htmlAst),
+      }
+    },
+  }
+}
+
+const collectHtmlUrlMentions = ({ url, htmlAst }) => {
+  const htmlUrlMentions = []
+  const addDependency = ({ type, node, attribute, specifier }) => {
+    const injected = Boolean(getHtmlNodeAttributeByName(node, "injected-by"))
+    const srcGeneratedFromInlineContent = Boolean(
+      getHtmlNodeAttributeByName(node, "src-generated-from-inline-content"),
+    )
+    let position
+    if (srcGeneratedFromInlineContent) {
+      // when generated from inline content,
+      // line, column is not "src" nor "content-src" but "original-position"
+      position = htmlNodePosition.readNodePosition(node)
+    } else {
+      position = htmlNodePosition.readAttributePosition(node, attribute.name)
+    }
+    const { line, column, originalLine, originalColumn } = position
+    htmlUrlMentions.push({
+      type,
+      htmlNode: node,
+      attribute,
+      injected,
+      srcGeneratedFromInlineContent,
+      specifier,
+      line,
+      column,
+      originalLine,
+      originalColumn,
+    })
+  }
+  const onNode = (node) => {
+    if (node.nodeName === "link") {
+      visitAttributeAsUrlSpecifier({
+        type: "link_href",
+        node,
+        attributeName: "href",
+      })
+      return
+    }
+    // if (node.nodeName === "style") {
+    //   // styles.push(node)
+    //   return
+    // }
+    if (node.nodeName === "script") {
+      visitAttributeAsUrlSpecifier({
+        type: "script_src",
+        node,
+        attributeName: "src",
+      })
+      return
+    }
+    if (node.nodeName === "a") {
+      visitAttributeAsUrlSpecifier({
+        type: "a_href",
+        node,
+        attributeName: "href",
+      })
+    }
+    if (node.nodeName === "iframe") {
+      visitAttributeAsUrlSpecifier({
+        type: "iframe_src",
+        node,
+        attributeName: "src",
+      })
+    }
+    if (node.nodeName === "img") {
+      visitAttributeAsUrlSpecifier({
+        type: "img_src",
+        node,
+        attributeName: "src",
+      })
+      visitSrcset({
+        type: "img_srcset",
+        node,
+      })
+      return
+    }
+    if (node.nodeName === "source") {
+      visitAttributeAsUrlSpecifier({
+        type: "source_src",
+        node,
+        attributeName: "src",
+      })
+      visitSrcset({
+        type: "source_srcset",
+        node,
+      })
+      return
+    }
+    // svg <image> tag
+    if (node.nodeName === "image") {
+      visitAttributeAsUrlSpecifier({
+        type: "image_href",
+        node,
+        attributeName: "href",
+      })
+      return
+    }
+    if (node.nodeName === "use") {
+      visitAttributeAsUrlSpecifier({
+        type: "use_href",
+        node,
+        attributeName: "href",
+      })
+      return
+    }
+  }
+  const visitAttributeAsUrlSpecifier = ({ type, node, attributeName }) => {
+    const attribute = getHtmlNodeAttributeByName(node, attributeName)
+    const value = attribute ? attribute.value : undefined
+    if (value) {
+      const inlinedBy = getHtmlNodeAttributeByName(node, "inlined-by")
+      if (inlinedBy) {
+        // during build the importmap is inlined
+        // and shoud not be considered as a dependency anymore
+        return
+      }
+      addDependency({
+        type,
+        node,
+        attribute,
+        specifier:
+          attributeName === "content-src" ? new URL(value, url).href : value,
+      })
+    } else if (attributeName === "src") {
+      visitAttributeAsUrlSpecifier({
+        type,
+        node,
+        attributeName: "content-src",
+      })
+    } else if (attributeName === "href") {
+      visitAttributeAsUrlSpecifier({
+        type,
+        node,
+        attributeName: "content-href",
+      })
+    }
+  }
+  const visitSrcset = ({ type, node }) => {
+    const srcsetAttribute = getHtmlNodeAttributeByName(node, "srcset")
+    const srcset = srcsetAttribute ? srcsetAttribute.value : undefined
+    if (srcset) {
+      const srcCandidates = htmlAttributeSrcSet.parse(srcset)
+      srcCandidates.forEach((srcCandidate) => {
+        addDependency({
+          type,
+          node,
+          attribute: srcsetAttribute,
+          specifier: srcCandidate.specifier,
+        })
+      })
+    }
+  }
+  visitHtmlAst(htmlAst, onNode)
+  return htmlUrlMentions
+}

package/src/omega/url_mentions/js_module_url_mentions.js

@@ -0,0 +1,91 @@
+import { collectProgramUrlMentions } from "@jsenv/utils/js_ast/program_url_mentions.js"
+import { applyBabelPlugins } from "@jsenv/utils/js_ast/apply_babel_plugins.js"
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+
+export const parseJsModuleUrlMentions = async ({
+  url,
+  generatedUrl,
+  content,
+}) => {
+  const { metadata } = await applyBabelPlugins({
+    babelPlugins: [
+      babelPluginMetadataUrlMentions,
+      babelPluginMetadataUsesTopLevelAwait,
+    ],
+    url,
+    generatedUrl,
+    content,
+  })
+  const { urlMentions, usesTopLevelAwait } = metadata
+  return {
+    urlMentions,
+    replaceUrls: async (getReplacement) => {
+      const magicSource = createMagicSource(content)
+      urlMentions.forEach((urlMention) => {
+        const replacement = getReplacement(urlMention)
+        if (replacement) {
+          const { start, end } = urlMention
+          magicSource.replace({
+            start,
+            end,
+            replacement,
+          })
+        }
+      })
+      return magicSource.toContentAndSourcemap()
+    },
+    data: {
+      usesTopLevelAwait,
+    },
+  }
+}
+
+const babelPluginMetadataUrlMentions = () => {
+  return {
+    name: "metadata-url-mentions",
+    visitor: {
+      Program(programPath, state) {
+        const urlMentions = []
+        collectProgramUrlMentions(programPath).forEach(
+          ({ type, subtype, path, specifierPath }) => {
+            const specifierNode = specifierPath.node
+            if (specifierNode.type === "StringLiteral") {
+              urlMentions.push({
+                type,
+                subtype,
+                path,
+                specifier: specifierNode.value,
+                start: specifierNode.start,
+                end: specifierNode.end,
+                line: specifierNode.loc.start.line,
+                column: specifierNode.loc.start.column,
+              })
+            }
+          },
+        )
+        state.file.metadata.urlMentions = urlMentions
+      },
+    },
+  }
+}
+
+const babelPluginMetadataUsesTopLevelAwait = () => {
+  return {
+    name: "metadata-uses-top-level-await",
+    visitor: {
+      Program: (programPath, state) => {
+        let usesTopLevelAwait = false
+        programPath.traverse({
+          AwaitExpression: (awaitPath) => {
+            const closestFunction = awaitPath.getFunctionParent()
+            if (!closestFunction) {
+              usesTopLevelAwait = true
+              awaitPath.stop()
+            }
+          },
+        })
+        state.file.metadata.usesTopLevelAwait = usesTopLevelAwait
+      },
+    },
+  }
+}

package/src/omega/url_mentions/parse_url_mentions.js

@@ -0,0 +1,37 @@
+import { parseHtmlUrlMentions } from "./html_url_mentions.js"
+import { parseCssUrlMentions } from "./css_url_mentions.js"
+import { parseJsModuleUrlMentions } from "./js_module_url_mentions.js"
+import { parseWorkerClassicUrlMentions } from "./worker_classic_url_mentions.js"
+
+const parsers = {
+  html: parseHtmlUrlMentions,
+  css: parseCssUrlMentions,
+  js_classic: (urlInfo) => {
+    if (urlInfo.subtype === "worker" || urlInfo.subtype === "service_worker") {
+      return parseWorkerClassicUrlMentions(urlInfo)
+    }
+    return {}
+  },
+  js_module: parseJsModuleUrlMentions,
+}
+
+export const parseUrlMentions = async ({ type, url, content, scenario }) => {
+  const parser = parsers[type]
+  if (!parser) {
+    return null
+  }
+  const {
+    urlMentions = [],
+    hotDecline = false,
+    hotAcceptSelf = false,
+    replaceUrls,
+    data,
+  } = await parser({ url, content, scenario })
+  return {
+    urlMentions,
+    hotDecline,
+    hotAcceptSelf,
+    replaceUrls,
+    data,
+  }
+}

package/src/omega/url_mentions/worker_classic_url_mentions.js

@@ -0,0 +1,37 @@
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+
+export const jsenvPluginImportScriptsBundler = () => {
+  return {
+    name: "jsenv:import_scripts_bundler",
+    bundle: {
+      js_classic: async (workerUrlInfos, { urlGraph }) => {
+        const bundleResult = {}
+        workerUrlInfos.forEach((workerUrlInfo) => {
+          const magicSource = createMagicSource(workerUrlInfo.content)
+          const visitDependencies = (urlInfo) => {
+            urlInfo.dependencies.forEach((dependencyUrl) => {
+              const dependencyUrlInfo = urlGraph.getUrlInfo(dependencyUrl)
+              // what if there was some sourcemap for this urlInfo?
+              // we should compose it too
+              magicSource.append(dependencyUrlInfo.content)
+              visitDependencies(dependencyUrlInfo)
+            })
+          }
+          visitDependencies(workerUrlInfo)
+          const { content, sourcemap } = magicSource.toContentAndSourcemap()
+          bundleResult[workerUrlInfo.url] = {
+            type: "worker_classic",
+            content,
+            sourcemap,
+          }
+        })
+        return bundleResult
+      },
+    },
+  }
+}
+
+export const parseWorkerClassicUrlMentions = () => {
+  // use babel_plugin_inline_worker_imports.js
+  return {}
+}