@jsenv/ast 1.1.0

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@jsenv/ast",
+  "version": "1.1.0",
+  "description": "Parse and transform HTML,CSS and JS",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jsenv/jsenv-core",
+    "directory": "packages/ast"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "author": {
+    "name": "dmail",
+    "email": "dmaillard06@gmail.com",
+    "url": "https://twitter.com/damienmaillard"
+  },
+  "engines": {
+    "node": ">=16.14.0"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./src/main.js"
+    },
+    "./*": "./*"
+  },
+  "files": [
+    "/src/"
+  ],
+  "scripts": {
+    "test": "node --conditions=development ./scripts/test.mjs"
+  },
+  "dependencies": {
+    "@babel/parser": "7.18.5",
+    "@parcel/css": "1.10.1",
+    "acorn": "8.7.1",
+    "acorn-import-assertions": "1.8.0",
+    "acorn-walk": "8.2.0",
+    "parse5": "7.0.0",
+    "postcss": "8.4.14",
+    "postcss-value-parser": "4.2.0"
+  }
+}

package/src/css/apply_post_css.js

@@ -0,0 +1,57 @@
+import { fileURLToPath } from "node:url"
+
+export const applyPostCss = async ({
+  sourcemaps = "comment",
+  plugins,
+  // https://github.com/postcss/postcss#options
+  options = {},
+  url,
+  map,
+  content,
+}) => {
+  const { default: postcss } = await import("postcss")
+
+  try {
+    const cssFileUrl = urlToFileUrl(url)
+    const result = await postcss(plugins).process(content, {
+      collectUrls: true,
+      from: fileURLToPath(cssFileUrl),
+      to: fileURLToPath(cssFileUrl),
+      map: {
+        annotation: sourcemaps === "file",
+        inline: sourcemaps === "inline",
+        // https://postcss.org/api/#sourcemapoptions
+        ...(map ? { prev: JSON.stringify(map) } : {}),
+      },
+      ...options,
+    })
+    return {
+      postCssMessages: result.messages,
+      map: result.map.toJSON(),
+      content: result.css,
+    }
+  } catch (error) {
+    if (error.name === "CssSyntaxError") {
+      console.error(String(error))
+      throw error
+    }
+    throw error
+  }
+}
+
+// the goal of this function is to take an url that is likely an http url
+// info a file:// url
+// for instance http://example.com/dir/file.js
+// must becomes file:///dir/file.js
+// but in windows it must be file://C:/dir/file.js
+const filesystemRootUrl =
+  process.platform === "win32" ? `file:///${process.cwd()[0]}:/` : "file:///"
+const urlToFileUrl = (url) => {
+  const urlString = String(url)
+  if (urlString.startsWith("file:")) {
+    return urlString
+  }
+  const origin = new URL(url).origin
+  const afterOrigin = urlString.slice(origin.length)
+  return new URL(afterOrigin, filesystemRootUrl).href
+}

package/src/css/parcel_css.js

@@ -0,0 +1,58 @@
+import { createRequire } from "node:module"
+import { fileURLToPath } from "node:url"
+
+const require = createRequire(import.meta.url)
+
+export const transpileWithParcel = (urlInfo, context) => {
+  const css = require("@parcel/css")
+  const targets = runtimeCompatToTargets(context.runtimeCompat)
+  const { code, map } = css.transform({
+    filename: fileURLToPath(urlInfo.originalUrl),
+    code: Buffer.from(urlInfo.content),
+    targets,
+    minify: false,
+  })
+  return { code, map }
+}
+
+export const minifyWithParcel = (urlInfo, context) => {
+  const css = require("@parcel/css")
+  const targets = runtimeCompatToTargets(context.runtimeCompat)
+  const { code, map } = css.transform({
+    filename: fileURLToPath(urlInfo.originalUrl),
+    code: Buffer.from(urlInfo.content),
+    targets,
+    minify: true,
+  })
+  return { code, map }
+}
+
+export const bundleWithParcel = (urlInfo, context) => {
+  const targets = runtimeCompatToTargets(context.runtimeCompat)
+  const css = require("@parcel/css")
+  const { code, map } = css.bundle({
+    filename: fileURLToPath(urlInfo.originalUrl),
+    targets,
+    minify: false,
+  })
+  return { code, map }
+}
+
+const runtimeCompatToTargets = (runtimeCompat) => {
+  const targets = {}
+  ;["chrome", "firefox", "ie", "opera", "safari"].forEach((runtimeName) => {
+    const version = runtimeCompat[runtimeName]
+    if (version) {
+      targets[runtimeName] = versionToBits(version)
+    }
+  })
+  return targets
+}
+
+const versionToBits = (version) => {
+  const [major, minor = 0, patch = 0] = version
+    .split("-")[0]
+    .split(".")
+    .map((v) => parseInt(v, 10))
+  return (major << 16) | (minor << 8) | patch
+}

package/src/css/postcss_plugin_url_visitor.js

@@ -0,0 +1,287 @@
+/**
+
+https://github.com/postcss/postcss/blob/master/docs/writing-a-plugin.md
+https://github.com/postcss/postcss/blob/master/docs/guidelines/plugin.md
+https://github.com/postcss/postcss/blob/master/docs/guidelines/runner.md#31-dont-show-js-stack-for-csssyntaxerror
+
+In case css sourcemap contains no%20source
+This is because of https://github.com/postcss/postcss/blob/fd30d3df5abc0954a0ec642a3cdc644ab2aacf9c/lib/map-generator.js#L231
+and it indicates a node has been replaced without passing source
+hence sourcemap cannot point the original source location
+
+*/
+
+import { pathToFileURL } from "node:url"
+
+import { require } from "../require.js"
+
+export const postCssPluginUrlVisitor = ({ urlVisitor = () => null }) => {
+  const parseCssValue = require("postcss-value-parser")
+  const stringifyCssNodes = parseCssValue.stringify
+
+  return {
+    postcssPlugin: "url_visitor",
+    prepare: (result) => {
+      const { from } = result.opts
+      const fromUrl = String(pathToFileURL(from))
+      const mutations = []
+      return {
+        AtRule: {
+          import: (atImportNode, { AtRule }) => {
+            if (atImportNode.parent.type !== "root") {
+              atImportNode.warn(result, "`@import` should be top level")
+              return
+            }
+            if (atImportNode.nodes) {
+              atImportNode.warn(
+                result,
+                "`@import` was not terminated correctly",
+              )
+              return
+            }
+            const parsed = parseCssValue(atImportNode.params)
+            let [urlNode] = parsed.nodes
+            if (
+              !urlNode ||
+              (urlNode.type !== "string" && urlNode.type !== "function")
+            ) {
+              atImportNode.warn(
+                result,
+                `No URL in \`${atImportNode.toString()}\``,
+              )
+              return
+            }
+            let url = ""
+            if (urlNode.type === "string") {
+              url = urlNode.value
+            } else if (urlNode.type === "function") {
+              // Invalid function
+              if (!/^url$/i.test(urlNode.value)) {
+                atImportNode.warn(
+                  result,
+                  `Invalid \`url\` function in \`${atImportNode.toString()}\``,
+                )
+                return
+              }
+              const firstNode = urlNode.nodes[0]
+              if (firstNode && firstNode.type === "string") {
+                urlNode = firstNode
+                url = urlNode.value
+              } else {
+                urlNode = urlNode.nodes
+                url = stringifyCssNodes(urlNode.nodes)
+              }
+            }
+
+            url = url.trim()
+            if (url.length === 0) {
+              atImportNode.warn(
+                result,
+                `Empty URL in \`${atImportNode.toString()}\``,
+              )
+              return
+            }
+
+            const specifier = url
+            url = new URL(specifier, fromUrl).href
+            if (url === fromUrl) {
+              atImportNode.warn(
+                result,
+                `\`@import\` loop in \`${atImportNode.toString()}\``,
+              )
+              return
+            }
+
+            const atRuleStart = atImportNode.source.start.offset
+            const atRuleEnd = atImportNode.source.end.offset + 1 // for the ";"
+            const atRuleRaw = atImportNode.source.input.css.slice(
+              atRuleStart,
+              atRuleEnd,
+            )
+            const specifierIndex = atRuleRaw.indexOf(atImportNode.params)
+            const specifierStart = atRuleStart + specifierIndex
+            const specifierEnd = specifierStart + atImportNode.params.length
+            const specifierLine = atImportNode.source.start.line
+            const specifierColumn =
+              atImportNode.source.start.column + specifierIndex
+            urlVisitor({
+              declarationNode: atImportNode,
+              type: "@import",
+              atRuleStart,
+              atRuleEnd,
+              specifier,
+              specifierLine,
+              specifierColumn,
+              specifierStart,
+              specifierEnd,
+              replace: (newUrlSpecifier) => {
+                if (newUrlSpecifier === urlNode.value) {
+                  return
+                }
+                urlNode.value = newUrlSpecifier
+                const newParams = parsed.toString()
+                const newAtImportRule = new AtRule({
+                  name: "import",
+                  params: newParams,
+                  source: atImportNode.source,
+                })
+                atImportNode.replaceWith(newAtImportRule)
+              },
+            })
+          },
+        },
+        Declaration: (declarationNode) => {
+          const parsed = parseCssValue(declarationNode.value)
+          const urlMutations = []
+          walkUrls(parsed, {
+            stringifyCssNodes,
+            visitor: ({ url, urlNode }) => {
+              // Empty URL
+              if (!urlNode || url.length === 0) {
+                declarationNode.warn(
+                  result,
+                  `Empty URL in \`${declarationNode.toString()}\``,
+                )
+                return
+              }
+              // Skip Data URI
+              if (isDataUrl(url)) {
+                return
+              }
+              const specifier = url
+              url = new URL(specifier, pathToFileURL(from))
+
+              const declarationNodeStart = declarationNode.source.start.offset
+              const afterDeclarationNode =
+                declarationNode.source.input.css.slice(declarationNodeStart)
+              const valueIndex = afterDeclarationNode.indexOf(
+                declarationNode.value,
+              )
+              const valueStart = declarationNodeStart + valueIndex
+              const specifierStart = valueStart + urlNode.sourceIndex
+              const specifierEnd =
+                specifierStart +
+                (urlNode.type === "word"
+                  ? urlNode.value.length
+                  : urlNode.value.length + 2) // the quotes
+              // value raw
+              // declarationNode.source.input.css.slice(valueStart)
+              // specifier raw
+              // declarationNode.source.input.css.slice(specifierStart, specifierEnd)
+              const specifierLine = declarationNode.source.start.line
+              const specifierColumn =
+                declarationNode.source.start.column +
+                (specifierStart - declarationNodeStart)
+
+              urlVisitor({
+                declarationNode,
+                type: "url",
+                specifier,
+                specifierLine,
+                specifierColumn,
+                specifierStart,
+                specifierEnd,
+                replace: (newUrlSpecifier) => {
+                  urlMutations.push(() => {
+                    // the specifier desires to be inside double quotes
+                    if (newUrlSpecifier[0] === `"`) {
+                      urlNode.type = "word"
+                      urlNode.value = newUrlSpecifier
+                      return
+                    }
+                    // the specifier desires to be inside simple quotes
+                    if (newUrlSpecifier[0] === `'`) {
+                      urlNode.type = "word"
+                      urlNode.value = newUrlSpecifier
+                      return
+                    }
+                    // the specifier desired to be just a word
+                    // for the "word" type so that newUrlSpecifier can opt-out of being between quotes
+                    // useful to inject __v__ calls for css inside js
+                    urlNode.type = "word"
+                    urlNode.value = newUrlSpecifier
+                  })
+                },
+              })
+            },
+          })
+          if (urlMutations.length) {
+            mutations.push(() => {
+              urlMutations.forEach((urlMutation) => {
+                urlMutation()
+              })
+              declarationNode.value = parsed.toString()
+            })
+          }
+        },
+        OnceExit: () => {
+          mutations.forEach((mutation) => {
+            mutation()
+          })
+        },
+      }
+    },
+  }
+}
+postCssPluginUrlVisitor.postcss = true
+
+const walkUrls = (parsed, { stringifyCssNodes, visitor }) => {
+  parsed.walk((node) => {
+    // https://github.com/andyjansson/postcss-functions
+    if (isUrlFunctionNode(node)) {
+      const { nodes } = node
+      const [urlNode] = nodes
+      const url =
+        urlNode && urlNode.type === "string"
+          ? urlNode.value
+          : stringifyCssNodes(nodes)
+      visitor({
+        url: url.trim(),
+        urlNode,
+      })
+      return
+    }
+
+    if (isImageSetFunctionNode(node)) {
+      Array.from(node.nodes).forEach((childNode) => {
+        if (childNode.type === "string") {
+          visitor({
+            url: childNode.value.trim(),
+            urlNode: childNode,
+          })
+          return
+        }
+
+        if (isUrlFunctionNode(node)) {
+          const { nodes } = childNode
+          const [urlNode] = nodes
+          const url =
+            urlNode && urlNode.type === "string"
+              ? urlNode.value
+              : stringifyCssNodes(nodes)
+          visitor({
+            url: url.trim(),
+            urlNode,
+          })
+          return
+        }
+      })
+    }
+  })
+}
+
+const isUrlFunctionNode = (node) => {
+  return node.type === "function" && /^url$/i.test(node.value)
+}
+
+const isImageSetFunctionNode = (node) => {
+  return (
+    node.type === "function" && /^(?:-webkit-)?image-set$/i.test(node.value)
+  )
+}
+
+const isDataUrl = (url) => {
+  return /data:[^\n\r;]+?(?:;charset=[^\n\r;]+?)?;base64,([\d+/A-Za-z]+={0,2})/.test(
+    url,
+  )
+}

package/src/html/html_analysis.js

@@ -0,0 +1,35 @@
+import { getHtmlNodeAttribute } from "./html_node_attributes.js"
+
+export const analyzeScriptNode = (scriptNode) => {
+  const type = getHtmlNodeAttribute(scriptNode, "type")
+  if (type === undefined || type === "text/javascript") {
+    return "classic"
+  }
+  if (type === "module") {
+    return "module"
+  }
+  if (type === "importmap") {
+    return "importmap"
+  }
+  return type
+}
+
+export const analyzeLinkNode = (linkNode) => {
+  const rel = getHtmlNodeAttribute(linkNode, "rel")
+  if (rel === "stylesheet") {
+    return {
+      isStylesheet: true,
+    }
+  }
+  const isRessourceHint = [
+    "preconnect",
+    "dns-prefetch",
+    "prefetch",
+    "preload",
+    "modulepreload",
+  ].includes(rel)
+  return {
+    isRessourceHint,
+    rel,
+  }
+}

package/src/html/html_inlining.js

@@ -0,0 +1,40 @@
+import {
+  getHtmlNodeAttribute,
+  setHtmlNodeAttributes,
+} from "./html_node_attributes.js"
+import { setHtmlNodeText } from "./html_node_text.js"
+
+export const inlineScriptNode = (script, textContent) => {
+  const src = getHtmlNodeAttribute(script, "src")
+  setHtmlNodeAttributes(script, {
+    "generated-from-src": src,
+    "src": undefined,
+    "crossorigin": undefined,
+    "integrity": undefined,
+  })
+  setHtmlNodeText(script, textContent)
+}
+
+export const inlineLinkStylesheetNode = (link, textContent) => {
+  const href = getHtmlNodeAttribute(link, "href")
+  setHtmlNodeAttributes(link, {
+    "generated-from-href": href,
+    "href": undefined,
+    "rel": undefined,
+    "type": undefined,
+    "as": undefined,
+    "crossorigin": undefined,
+    "integrity": undefined,
+  })
+  link.nodeName = "style"
+  link.tagName = "style"
+  setHtmlNodeText(link, textContent)
+}
+
+export const inlineImgNode = (img, contentAsBase64) => {
+  const src = getHtmlNodeAttribute(img, "src")
+  setHtmlNodeAttributes(img, {
+    "generated-from-src": src,
+    "src": contentAsBase64,
+  })
+}

package/src/html/html_node.js

@@ -0,0 +1,112 @@
+import { parseFragment } from "parse5"
+
+import {
+  getHtmlNodeAttribute,
+  setHtmlNodeAttributes,
+} from "./html_node_attributes.js"
+import { findHtmlNode } from "./html_search.js"
+import { analyzeScriptNode } from "./html_analysis.js"
+
+export const removeHtmlNode = (htmlNode) => {
+  const { childNodes } = htmlNode.parentNode
+  childNodes.splice(childNodes.indexOf(htmlNode), 1)
+}
+
+export const createHtmlNode = ({ tagName, textContent = "", ...rest }) => {
+  const html = `<${tagName} ${stringifyAttributes(
+    rest,
+  )}>${textContent}</${tagName}>`
+  const fragment = parseFragment(html)
+  return fragment.childNodes[0]
+}
+
+export const injectScriptNodeAsEarlyAsPossible = (htmlAst, scriptNode) => {
+  const injectedBy = getHtmlNodeAttribute(scriptNode, "injected-by")
+  if (injectedBy === undefined) {
+    setHtmlNodeAttributes(scriptNode, {
+      "injected-by": "jsenv",
+    })
+  }
+  const isModule = analyzeScriptNode(scriptNode) === "module"
+  if (isModule) {
+    const firstImportmapScript = findHtmlNode(htmlAst, (node) => {
+      if (node.nodeName !== "script") return false
+      return analyzeScriptNode(node) === "importmap"
+    })
+    if (firstImportmapScript) {
+      return insertAfter(
+        scriptNode,
+        firstImportmapScript.parentNode,
+        firstImportmapScript,
+      )
+    }
+  }
+  const headNode = findChild(htmlAst, (node) => node.nodeName === "html")
+    .childNodes[0]
+  const firstHeadScript = findChild(headNode, (node) => {
+    return node.nodeName === "script"
+  })
+  return insertBefore(scriptNode, headNode, firstHeadScript)
+}
+
+const insertBefore = (nodeToInsert, futureParentNode, futureNextSibling) => {
+  const { childNodes = [] } = futureParentNode
+  const futureIndex = futureNextSibling
+    ? childNodes.indexOf(futureNextSibling)
+    : 0
+  injectWithWhitespaces(nodeToInsert, futureParentNode, futureIndex)
+}
+
+const insertAfter = (nodeToInsert, futureParentNode, futurePrevSibling) => {
+  const { childNodes = [] } = futureParentNode
+  const futureIndex = futurePrevSibling
+    ? childNodes.indexOf(futurePrevSibling) + 1
+    : childNodes.length
+  injectWithWhitespaces(nodeToInsert, futureParentNode, futureIndex)
+}
+
+const injectWithWhitespaces = (nodeToInsert, futureParentNode, futureIndex) => {
+  const { childNodes = [] } = futureParentNode
+  const previousSiblings = childNodes.slice(0, futureIndex)
+  const nextSiblings = childNodes.slice(futureIndex)
+  const futureChildNodes = []
+  const previousSibling = previousSiblings[0]
+  if (previousSibling) {
+    futureChildNodes.push(...previousSiblings)
+  }
+  if (!previousSibling || previousSibling.nodeName !== "#text") {
+    futureChildNodes.push({
+      nodeName: "#text",
+      value: "\n    ",
+      parentNode: futureParentNode,
+    })
+  }
+  futureChildNodes.push(nodeToInsert)
+  const nextSibling = nextSiblings[0]
+  if (!nextSibling || nextSibling.nodeName !== "#text") {
+    futureChildNodes.push({
+      nodeName: "#text",
+      value: "\n    ",
+      parentNode: futureParentNode,
+    })
+  }
+  if (nextSibling) {
+    futureChildNodes.push(...nextSiblings)
+  }
+  futureParentNode.childNodes = futureChildNodes
+}
+
+const findChild = ({ childNodes = [] }, predicate) => childNodes.find(predicate)
+
+const stringifyAttributes = (object) => {
+  return Object.keys(object)
+    .map((key) => `${key}=${valueToHtmlAttributeValue(object[key])}`)
+    .join(" ")
+}
+
+const valueToHtmlAttributeValue = (value) => {
+  if (typeof value === "string") {
+    return JSON.stringify(value)
+  }
+  return `"${JSON.stringify(value)}"`
+}

package/src/html/html_node_attributes.js

@@ -0,0 +1,43 @@
+export const getHtmlNodeAttribute = (htmlNode, attributeName) => {
+  const attribute = getHtmlAttributeByName(htmlNode, attributeName)
+  return attribute ? attribute.value || "" : undefined
+}
+
+export const setHtmlNodeAttributes = (htmlNode, attributesToAssign) => {
+  if (typeof attributesToAssign !== "object") {
+    throw new TypeError(`attributesToAssign must be an object`)
+  }
+  const { attrs } = htmlNode
+  if (!attrs) return
+  Object.keys(attributesToAssign).forEach((key) => {
+    const existingAttributeIndex = attrs.findIndex(({ name }) => name === key)
+    const value = attributesToAssign[key]
+    // remove no-op
+    if (existingAttributeIndex === -1 && value === undefined) {
+      return
+    }
+    // add
+    if (existingAttributeIndex === -1 && value !== undefined) {
+      attrs.push({
+        name: key,
+        value,
+      })
+      return
+    }
+    // remove
+    if (value === undefined) {
+      attrs.splice(existingAttributeIndex, 1)
+      return
+    }
+    // update
+    attrs[existingAttributeIndex].value = value
+  })
+}
+
+const getHtmlAttributeByName = (htmlNode, attributeName) => {
+  const attrs = htmlNode.attrs
+  const attribute = attrs
+    ? attrs.find((attr) => attr.name === attributeName)
+    : null
+  return attribute
+}