@jsenv/ast 1.1.0

package/src/js/js_static_analysis/new_url.js

@@ -0,0 +1,132 @@
+import { isStringLiteralNode } from "./helpers.js"
+
+export const isNewUrlCall = (node) => {
+  return (
+    node.type === "NewExpression" &&
+    node.callee.type === "Identifier" &&
+    node.callee.name === "URL"
+  )
+}
+export const analyzeNewUrlCall = (node, { isJsModule, onUrl }) => {
+  if (node.arguments.length === 1) {
+    const firstArgNode = node.arguments[0]
+    const urlType = analyzeUrlNodeType(firstArgNode, { isJsModule })
+    if (urlType === "StringLiteral") {
+      const specifierNode = firstArgNode
+      onUrl({
+        type: "js_url_specifier",
+        subtype: "new_url_first_arg",
+        specifier: specifierNode.value,
+        specifierStart: specifierNode.start,
+        specifierEnd: specifierNode.end,
+        specifierLine: specifierNode.loc.start.line,
+        specifierColumn: specifierNode.loc.start.column,
+      })
+    }
+    return
+  }
+  if (node.arguments.length === 2) {
+    const firstArgNode = node.arguments[0]
+    const secondArgNode = node.arguments[1]
+    const baseUrlType = analyzeUrlNodeType(secondArgNode, { isJsModule })
+    if (baseUrlType) {
+      // we can understand the second argument
+      const urlType = analyzeUrlNodeType(firstArgNode, { isJsModule })
+      if (urlType === "StringLiteral") {
+        // we can understand the first argument
+        const specifierNode = firstArgNode
+        onUrl({
+          type: "js_url_specifier",
+          subtype: "new_url_first_arg",
+          specifier: specifierNode.value,
+          specifierStart: specifierNode.start,
+          specifierEnd: specifierNode.end,
+          specifierLine: specifierNode.loc.start.line,
+          specifierColumn: specifierNode.loc.start.column,
+          baseUrlType,
+          baseUrl:
+            baseUrlType === "StringLiteral" ? secondArgNode.value : undefined,
+        })
+      }
+      if (baseUrlType === "StringLiteral") {
+        const specifierNode = secondArgNode
+        onUrl({
+          type: "js_url_specifier",
+          subtype: "new_url_second_arg",
+          specifier: specifierNode.value,
+          specifierStart: specifierNode.start,
+          specifierEnd: specifierNode.end,
+          specifierLine: specifierNode.loc.start.line,
+          specifierColumn: specifierNode.loc.start.column,
+        })
+      }
+    }
+  }
+}
+
+const analyzeUrlNodeType = (secondArgNode, { isJsModule }) => {
+  if (isStringLiteralNode(secondArgNode)) {
+    return "StringLiteral"
+  }
+  if (isImportMetaUrl(secondArgNode)) {
+    return "import.meta.url"
+  }
+  if (isWindowOrigin(secondArgNode)) {
+    return "window.origin"
+  }
+  if (!isJsModule && isContextMetaUrlFromSystemJs(secondArgNode)) {
+    return "context.meta.url"
+  }
+  if (!isJsModule && isDocumentCurrentScriptSrc(secondArgNode)) {
+    return "document.currentScript.src"
+  }
+  return null
+}
+
+const isImportMetaUrl = (node) => {
+  return (
+    node.type === "MemberExpression" &&
+    node.object.type === "MetaProperty" &&
+    node.property.type === "Identifier" &&
+    node.property.name === "url"
+  )
+}
+
+const isWindowOrigin = (node) => {
+  return (
+    node.type === "MemberExpression" &&
+    node.object.type === "Identifier" &&
+    node.object.name === "window" &&
+    node.property.type === "Identifier" &&
+    node.property.name === "origin"
+  )
+}
+
+const isContextMetaUrlFromSystemJs = (node) => {
+  return (
+    node.type === "MemberExpression" &&
+    node.object.type === "MemberExpression" &&
+    node.object.object.type === "Identifier" &&
+    // because of minification we can't assume _context.
+    // so anything matching "*.meta.url" (in the context of new URL())
+    // will be assumed to be the equivalent to "import.meta.url"
+    // node.object.object.name === "_context" &&
+    node.object.property.type === "Identifier" &&
+    node.object.property.name === "meta" &&
+    node.property.type === "Identifier" &&
+    node.property.name === "url"
+  )
+}
+
+const isDocumentCurrentScriptSrc = (node) => {
+  return (
+    node.type === "MemberExpression" &&
+    node.object.type === "MemberExpression" &&
+    node.object.object.type === "Identifier" &&
+    node.object.object.name === "document" &&
+    node.object.property.type === "Identifier" &&
+    node.object.property.name === "currentScript" &&
+    node.property.type === "Identifier" &&
+    node.property.name === "src"
+  )
+}

package/src/js/js_static_analysis/system.js

@@ -0,0 +1,74 @@
+import { isStringLiteralNode } from "./helpers.js"
+
+export const isSystemRegisterCall = (node) => {
+  const callee = node.callee
+  return (
+    callee.type === "MemberExpression" &&
+    callee.object.type === "Identifier" &&
+    callee.object.name === "System" &&
+    callee.property.type === "Identifier" &&
+    callee.property.name === "register"
+  )
+}
+export const analyzeSystemRegisterCall = (node, { onUrl }) => {
+  const firstArgNode = node.arguments[0]
+  if (firstArgNode.type === "ArrayExpression") {
+    analyzeSystemRegisterDeps(firstArgNode, { onUrl })
+    return
+  }
+  if (isStringLiteralNode(firstArgNode)) {
+    const secondArgNode = node.arguments[1]
+    if (secondArgNode.type === "ArrayExpression") {
+      analyzeSystemRegisterDeps(secondArgNode, { onUrl })
+      return
+    }
+  }
+}
+const analyzeSystemRegisterDeps = (node, { onUrl }) => {
+  const elements = node.elements
+  elements.forEach((element) => {
+    if (isStringLiteralNode(element)) {
+      const specifierNode = element
+      onUrl({
+        type: "js_url_specifier",
+        subtype: "system_register_arg",
+        expectedType: "js_classic",
+        specifier: specifierNode.value,
+        specifierStart: specifierNode.start,
+        specifierEnd: specifierNode.end,
+        specifierLine: specifierNode.loc.start.line,
+        specifierColumn: specifierNode.loc.start.column,
+      })
+    }
+  })
+}
+
+export const isSystemImportCall = (node) => {
+  const callee = node.callee
+  return (
+    callee.type === "MemberExpression" &&
+    callee.object.type === "Identifier" &&
+    // because of minification we can't assume _context.
+    // so anything matching "*.import()"
+    // will be assumed to be the equivalent to "import()"
+    // callee.object.name === "_context" &&
+    callee.property.type === "Identifier" &&
+    callee.property.name === "import"
+  )
+}
+export const analyzeSystemImportCall = (node, { onUrl }) => {
+  const firstArgNode = node.arguments[0]
+  if (isStringLiteralNode(firstArgNode)) {
+    const specifierNode = firstArgNode
+    onUrl({
+      type: "js_url_specifier",
+      subtype: "system_import_arg",
+      expectedType: "js_classic",
+      specifier: specifierNode.value,
+      specifierStart: specifierNode.start,
+      specifierEnd: specifierNode.end,
+      specifierLine: specifierNode.loc.start.line,
+      specifierColumn: specifierNode.loc.start.column,
+    })
+  }
+}

package/src/js/js_static_analysis/web_worker.js

@@ -0,0 +1,32 @@
+import { isStringLiteralNode } from "./helpers.js"
+
+export const isImportScriptsCall = (node) => {
+  const callee = node.callee
+  if (callee.type === "Identifier" && callee.name === "importScripts") {
+    return true
+  }
+  return (
+    callee.type === "MemberExpression" &&
+    callee.object.type === "Identifier" &&
+    callee.object.name === "self" &&
+    callee.property.type === "Identifier" &&
+    callee.property.name === "importScripts"
+  )
+}
+export const analyzeImportScriptCalls = (node, { onUrl }) => {
+  node.arguments.forEach((arg) => {
+    if (isStringLiteralNode(arg)) {
+      const specifierNode = arg
+      onUrl({
+        type: "js_url_specifier",
+        subtype: "self_import_scripts_arg",
+        expectedType: "js_classic",
+        specifier: specifierNode.value,
+        specifierStart: specifierNode.start,
+        specifierEnd: specifierNode.end,
+        specifierLine: specifierNode.loc.start.line,
+        specifierColumn: specifierNode.loc.start.column,
+      })
+    }
+  })
+}

package/src/js/js_static_analysis/web_worker_entry_point.js

@@ -0,0 +1,143 @@
+import { getTypePropertyNode, isStringLiteralNode } from "./helpers.js"
+import { isNewUrlCall, analyzeNewUrlCall } from "./new_url.js"
+
+export const isNewWorkerCall = (node) => {
+  return (
+    node.type === "NewExpression" &&
+    node.callee.type === "Identifier" &&
+    node.callee.name === "Worker"
+  )
+}
+export const analyzeNewWorkerCall = (node, { isJsModule, onUrl }) => {
+  analyzeWorkerCallArguments(node, {
+    isJsModule,
+    onUrl,
+    referenceSubtype: "new_worker_first_arg",
+    expectedSubtype: "worker",
+  })
+}
+
+export const isNewSharedWorkerCall = (node) => {
+  return (
+    node.type === "NewExpression" &&
+    node.callee.type === "Identifier" &&
+    node.callee.name === "SharedWorker"
+  )
+}
+export const analyzeNewSharedWorkerCall = (node, { isJsModule, onUrl }) => {
+  analyzeWorkerCallArguments(node, {
+    isJsModule,
+    onUrl,
+    referenceSubtype: "new_shared_worker_first_arg",
+    expectedSubtype: "shared_worker",
+  })
+}
+
+export const isServiceWorkerRegisterCall = (node) => {
+  if (node.type !== "CallExpression") {
+    return false
+  }
+  const callee = node.callee
+  if (
+    callee.type === "MemberExpression" &&
+    callee.property.type === "Identifier" &&
+    callee.property.name === "register"
+  ) {
+    const parentObject = callee.object
+    if (parentObject.type === "MemberExpression") {
+      const parentProperty = parentObject.property
+      if (
+        parentProperty.type === "Identifier" &&
+        parentProperty.name === "serviceWorker"
+      ) {
+        const grandParentObject = parentObject.object
+        if (grandParentObject.type === "MemberExpression") {
+          // window.navigator.serviceWorker.register
+          const grandParentProperty = grandParentObject.property
+          if (
+            grandParentProperty.type === "Identifier" &&
+            grandParentProperty.name === "navigator"
+          ) {
+            const ancestorObject = grandParentObject.object
+            if (
+              ancestorObject.type === "Identifier" &&
+              ancestorObject.name === "window"
+            ) {
+              return true
+            }
+          }
+        }
+        if (grandParentObject.type === "Identifier") {
+          // navigator.serviceWorker.register
+          if (grandParentObject.name === "navigator") {
+            return true
+          }
+        }
+      }
+    }
+  }
+  return false
+}
+export const analyzeServiceWorkerRegisterCall = (
+  node,
+  { isJsModule, onUrl },
+) => {
+  analyzeWorkerCallArguments(node, {
+    isJsModule,
+    onUrl,
+    referenceSubtype: "service_worker_register_first_arg",
+    expectedSubtype: "service_worker",
+  })
+}
+
+const analyzeWorkerCallArguments = (
+  node,
+  { isJsModule, onUrl, referenceSubtype, expectedSubtype },
+) => {
+  let expectedType = "js_classic"
+  let typePropertyNode
+  const secondArgNode = node.arguments[1]
+  if (secondArgNode) {
+    typePropertyNode = getTypePropertyNode(secondArgNode)
+    if (typePropertyNode) {
+      const typePropertyValueNode = typePropertyNode.value
+      if (isStringLiteralNode(typePropertyValueNode)) {
+        const typePropertyValue = typePropertyValueNode.value
+        if (typePropertyValue === "module") {
+          expectedType = "js_module"
+        }
+      }
+    }
+  }
+
+  const firstArgNode = node.arguments[0]
+  if (isStringLiteralNode(firstArgNode)) {
+    const specifierNode = firstArgNode
+    onUrl({
+      type: "js_url_specifier",
+      subtype: referenceSubtype,
+      expectedType,
+      expectedSubtype,
+      typePropertyNode,
+      specifier: specifierNode.value,
+      specifierStart: specifierNode.start,
+      specifierEnd: specifierNode.end,
+      specifierLine: specifierNode.loc.start.line,
+      specifierColumn: specifierNode.loc.start.column,
+    })
+    return
+  }
+  if (isNewUrlCall(firstArgNode)) {
+    analyzeNewUrlCall(firstArgNode, {
+      isJsModule,
+      onUrl: (mention) => {
+        Object.assign(mention, {
+          expectedType,
+          expectedSubtype,
+          typePropertyNode,
+        })
+        onUrl(mention)
+      },
+    })
+  }
+}

package/src/js/parse_js_urls.js

@@ -0,0 +1,122 @@
+import { ancestor } from "acorn-walk"
+
+import { parseJsWithAcorn } from "./parse_js_with_acorn.js"
+import {
+  analyzeImportExpression,
+  analyzeImportDeclaration,
+  analyzeExportNamedDeclaration,
+  analyzeExportAllDeclaration,
+} from "./js_static_analysis/import_export.js"
+import {
+  isNewWorkerCall,
+  analyzeNewWorkerCall,
+  isNewSharedWorkerCall,
+  analyzeNewSharedWorkerCall,
+  isServiceWorkerRegisterCall,
+  analyzeServiceWorkerRegisterCall,
+} from "./js_static_analysis/web_worker_entry_point.js"
+import {
+  isImportScriptsCall,
+  analyzeImportScriptCalls,
+} from "./js_static_analysis/web_worker.js"
+import {
+  isNewUrlCall,
+  analyzeNewUrlCall,
+} from "./js_static_analysis/new_url.js"
+import {
+  isSystemRegisterCall,
+  analyzeSystemRegisterCall,
+  isSystemImportCall,
+  analyzeSystemImportCall,
+} from "./js_static_analysis/system.js"
+
+export const parseJsUrls = async ({
+  js,
+  url,
+  isJsModule = false,
+  isWebWorker = false,
+} = {}) => {
+  const jsUrls = []
+  const jsAst = await parseJsWithAcorn({
+    js,
+    url,
+    isJsModule,
+  })
+  const onUrl = (jsUrl) => {
+    jsUrls.push(jsUrl)
+  }
+  ancestor(jsAst, {
+    ImportDeclaration: (node) => {
+      analyzeImportDeclaration(node, { onUrl })
+    },
+    ImportExpression: (node) => {
+      analyzeImportExpression(node, { onUrl })
+    },
+    ExportNamedDeclaration: (node) => {
+      analyzeExportNamedDeclaration(node, { onUrl })
+    },
+    ExportAllDeclaration: (node) => {
+      analyzeExportAllDeclaration(node, { onUrl })
+    },
+    CallExpression: (node) => {
+      if (isServiceWorkerRegisterCall(node)) {
+        analyzeServiceWorkerRegisterCall(node, {
+          isJsModule,
+          onUrl,
+        })
+        return
+      }
+      if (isWebWorker && isImportScriptsCall(node)) {
+        analyzeImportScriptCalls(node, {
+          onUrl,
+        })
+        return
+      }
+      if (!isJsModule && isSystemRegisterCall(node)) {
+        analyzeSystemRegisterCall(node, {
+          onUrl,
+        })
+        return
+      }
+      if (!isJsModule && isSystemImportCall(node)) {
+        analyzeSystemImportCall(node, {
+          onUrl,
+        })
+        return
+      }
+    },
+    NewExpression: (node, ancestors) => {
+      if (isNewWorkerCall(node)) {
+        analyzeNewWorkerCall(node, {
+          isJsModule,
+          onUrl,
+        })
+        return
+      }
+      if (isNewSharedWorkerCall(node)) {
+        analyzeNewSharedWorkerCall(node, {
+          isJsModule,
+          onUrl,
+        })
+        return
+      }
+      if (isNewUrlCall(node)) {
+        const parent = ancestors[ancestors.length - 2]
+        if (
+          parent &&
+          (isNewWorkerCall(parent) ||
+            isNewSharedWorkerCall(parent) ||
+            isServiceWorkerRegisterCall(parent))
+        ) {
+          return
+        }
+        analyzeNewUrlCall(node, {
+          isJsModule,
+          onUrl,
+        })
+        return
+      }
+    },
+  })
+  return jsUrls
+}

package/src/js/parse_js_with_acorn.js

@@ -0,0 +1,40 @@
+import { createJsParseError } from "./js_parse_error.js"
+
+let AcornParser
+let _getLineInfo
+
+export const parseJsWithAcorn = async ({ js, url, isJsModule }) => {
+  await initAcornParser()
+  try {
+    // https://github.com/acornjs/acorn/tree/master/acorn#interface
+    const jsAst = AcornParser.parse(js, {
+      locations: true,
+      allowAwaitOutsideFunction: true,
+      sourceType: isJsModule ? "module" : "script",
+      ecmaVersion: 2022,
+    })
+    return jsAst
+  } catch (e) {
+    if (e && e.name === "SyntaxError") {
+      const { line, column } = _getLineInfo(js, e.raisedAt)
+      throw createJsParseError({
+        message: e.message,
+        url,
+        line,
+        column,
+      })
+    }
+    throw e
+  }
+}
+
+const initAcornParser = async () => {
+  if (AcornParser) {
+    return
+  }
+  const { Parser, getLineInfo } = await import("acorn")
+  const { importAssertions } = await import("acorn-import-assertions")
+
+  AcornParser = Parser.extend(importAssertions)
+  _getLineInfo = getLineInfo
+}

package/src/main.js

@@ -0,0 +1,44 @@
+// html
+export {
+  parseHtmlString,
+  stringifyHtmlAst,
+  parseSvgString,
+  stringifySvgAst,
+} from "./html/html_parse.js"
+export { visitHtmlNodes, findHtmlNode } from "./html/html_search.js"
+export { analyzeScriptNode, analyzeLinkNode } from "./html/html_analysis.js"
+export {
+  getHtmlNodeText,
+  removeHtmlNodeText,
+  setHtmlNodeText,
+} from "./html/html_node_text.js"
+export { parseSrcSet, stringifySrcSet } from "./html/html_src_set.js"
+export {
+  getHtmlNodeAttribute,
+  setHtmlNodeAttributes,
+} from "./html/html_node_attributes.js"
+export {
+  removeHtmlNode,
+  createHtmlNode,
+  injectScriptNodeAsEarlyAsPossible,
+} from "./html/html_node.js"
+export {
+  inlineScriptNode,
+  inlineLinkStylesheetNode,
+  inlineImgNode,
+} from "./html/html_inlining.js"
+export {
+  getHtmlNodePosition,
+  getHtmlNodeAttributePosition,
+} from "./html/html_node_position.js"
+
+// css
+export { applyPostCss } from "./css/apply_post_css.js"
+export { transpileWithParcel, minifyWithParcel } from "./css/parcel_css.js"
+export { postCssPluginUrlVisitor } from "./css/postcss_plugin_url_visitor.js"
+
+// js
+export { applyBabelPlugins } from "./js/apply_babel_plugins.js"
+export { injectJsImport } from "./js/babel_utils.js"
+export { parseJsWithAcorn } from "./js/parse_js_with_acorn.js"
+export { parseJsUrls } from "./js/parse_js_urls.js"

package/src/require.js

@@ -0,0 +1,3 @@
+import { createRequire } from "node:module"
+
+export const require = createRequire(import.meta.url)