@jsenv/ast 5.0.1 → 5.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsenv/ast",
-  "version": "5.0.1",
+  "version": "5.1.0",
   "description": "Parse and transform HTML,CSS and JS",
   "license": "MIT",
   "repository": {
@@ -34,11 +34,12 @@
   },
   "dependencies": {
     "@babel/parser": "7.22.7",
+    "@jsenv/urls": "2.2.0",
     "acorn": "8.10.0",
     "acorn-import-assertions": "1.9.0",
     "acorn-walk": "8.2.0",
     "parse5": "7.1.2",
-    "postcss": "8.4.26",
+    "postcss": "8.4.27",
     "postcss-value-parser": "4.2.0"
   }
 }

package/src/html/html_analysis.js

@@ -6,18 +6,21 @@ export const analyzeScriptNode = (scriptNode) => {
     return {
       type: "js_classic",
       contentType: "text/javascript",
+      extension: ".js",
     };
   }
   if (typeAttribute === "module") {
     return {
       type: "js_module",
       contentType: "text/javascript",
+      extension: ".js",
     };
   }
   if (typeAttribute === "importmap") {
     return {
       type: "importmap",
       contentType: "application/importmap+json",
+      extension: ".importmap",
     };
   }
   // jsx
@@ -74,6 +77,7 @@ export const analyzeScriptNode = (scriptNode) => {
   return {
     type: "text",
     contentType: "text/plain",
+    extension: ".txt",
   };
 };
 

package/src/html/html_inline_content_url.js

@@ -0,0 +1,50 @@
+import { injectQueryParams, urlToExtension, urlToBasename } from "@jsenv/urls";
+
+import { generateUrlForInlineContent } from "../inline_content_url.js";
+import { getHtmlNodeAttribute } from "./html_node_attributes.js";
+import { getHtmlNodePosition } from "./html_node_position.js";
+import { analyzeScriptNode } from "./html_analysis.js";
+
+export const getUrlForContentInsideHtml = (node, { htmlUrl, url }) => {
+  let externalSpecifierAttributeName;
+  let basename;
+  let extension;
+
+  if (node.nodeName === "script") {
+    externalSpecifierAttributeName = "inlined-from-src";
+    const scriptAnalysisResult = analyzeScriptNode(node);
+    extension = scriptAnalysisResult.extension;
+  } else if (node.nodeName === "style") {
+    externalSpecifierAttributeName = "inlined-from-href";
+    extension = ".css";
+  } else if (node.nodeName === "link") {
+    basename = urlToBasename(url);
+    extension = urlToExtension(url);
+  }
+
+  if (externalSpecifierAttributeName) {
+    const externalSpecifier = getHtmlNodeAttribute(
+      node,
+      externalSpecifierAttributeName,
+    );
+    if (externalSpecifier) {
+      const inlineContentUrl = injectQueryParams(externalSpecifier, {
+        inlined: "",
+      });
+      return inlineContentUrl;
+    }
+  }
+  const { line, column, lineEnd, columnEnd } = getHtmlNodePosition(node, {
+    preferOriginal: true,
+  });
+  const inlineContentUrl = generateUrlForInlineContent({
+    url: htmlUrl,
+    basename,
+    extension,
+    line,
+    column,
+    lineEnd,
+    columnEnd,
+  });
+  return inlineContentUrl;
+};

package/src/inline_content_url.js

@@ -0,0 +1,33 @@
+import { urlToFilename, urlToExtension } from "@jsenv/urls";
+
+export const generateUrlForInlineContent = ({
+  url,
+  extension,
+  basename,
+  line,
+  column,
+  lineEnd,
+  columnEnd,
+}) => {
+  if (extension === undefined) {
+    extension = urlToExtension(url);
+  }
+
+  let generatedName = "";
+  if (basename !== undefined) {
+    generatedName += basename;
+  }
+  if (line !== undefined && column !== undefined) {
+    generatedName += `L${line}C${column}`;
+    if (lineEnd !== undefined && columnEnd !== undefined) {
+      generatedName += `-L${lineEnd}C${columnEnd}`;
+    }
+  }
+
+  const filenameRaw = urlToFilename(url);
+  const filename = `${filenameRaw}@${generatedName}${extension}`;
+  // ideally we should keep query params from url
+  // maybe we could use a custom scheme like "inline:"
+  const inlineContentUrl = new URL(filename, url).href;
+  return inlineContentUrl;
+};

package/src/js/apply_babel_plugins.js

@@ -3,7 +3,7 @@
  * - https://astexplorer.net/
  * - https://bvaughn.github.io/babel-repl
  */
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 
 import { createJsParseError } from "./js_parse_error.js";
 
@@ -63,10 +63,10 @@ export const applyBabelPlugins = async ({
   try {
     if (ast) {
       const result = await transformFromAstAsync(ast, input, options);
-      return result;
+      return normalizeResult(result);
     }
     const result = await transformAsync(input, options);
-    return result;
+    return normalizeResult(result);
   } catch (error) {
     if (error && error.code === "BABEL_PARSE_ERROR") {
       throw createJsParseError({
@@ -82,6 +82,16 @@ export const applyBabelPlugins = async ({
   }
 };
 
+const normalizeResult = (result) => {
+  const { map } = result;
+  if (map) {
+    map.sources.forEach((source, index) => {
+      map.sources[index] = pathToFileURL(source).href;
+    });
+  }
+  return result;
+};
+
 const useTypeScriptExtension = (url) => {
   const { pathname } = new URL(url);
   return pathname.endsWith(".ts") || pathname.endsWith(".tsx");

package/src/js/js_inline_content_url.js

@@ -0,0 +1,19 @@
+import { injectQueryParams } from "@jsenv/urls";
+import { CONTENT_TYPE } from "@jsenv/utils/src/content_type/content_type.js";
+
+import { generateUrlForInlineContent } from "../inline_content_url.js";
+
+export const getUrlForContentInsideJs = (inlineReferenceInfo, { url }) => {
+  const { inlinedFromUrl } = inlineReferenceInfo;
+  if (inlinedFromUrl) {
+    return injectQueryParams(inlinedFromUrl, { inlined: "" });
+  }
+  return generateUrlForInlineContent({
+    url,
+    extension: CONTENT_TYPE.asFileExtension(inlineReferenceInfo.contentType),
+    line: inlineReferenceInfo.line,
+    column: inlineReferenceInfo.column,
+    lineEnd: inlineReferenceInfo.lineEnd,
+    columnEnd: inlineReferenceInfo.columnEnd,
+  });
+};

package/src/js/js_static_analysis/helpers.js

@@ -1,4 +1,4 @@
-export const getTypePropertyNode = (node) => {
+export const findPropertyNodeByName = (node, name) => {
   if (node.type !== "ObjectExpression") {
     return null;
   }
@@ -7,7 +7,7 @@ export const getTypePropertyNode = (node) => {
     return (
       property.type === "Property" &&
       property.key.type === "Identifier" &&
-      property.key.name === "type"
+      property.key.name === name
     );
   });
 };

package/src/js/js_static_analysis/json_parse.js

@@ -11,13 +11,17 @@ export const isJSONParseCall = (node) => {
   );
 };
 
-export const analyzeJSONParseCall = (node, { onInlineContent }) => {
+export const analyzeJSONParseCall = (
+  node,
+  { onInlineContent, readInlinedFromUrl },
+) => {
   const nodeHoldingContent = node.arguments[0];
   const contentInfo = extractContentInfo(nodeHoldingContent);
   if (contentInfo) {
     onInlineContent({
       type: "json_parse_first_arg",
       contentType: "application/json",
+      inlinedFromUrl: readInlinedFromUrl(node),
       start: nodeHoldingContent.start,
       end: nodeHoldingContent.end,
       line: nodeHoldingContent.loc.start.line,

package/src/js/js_static_analysis/new_blob.js

@@ -1,6 +1,6 @@
 import {
   isStringLiteralNode,
-  getTypePropertyNode,
+  findPropertyNodeByName,
   extractContentInfo,
 } from "./helpers.js";
 
@@ -11,7 +11,10 @@ export const isNewBlobCall = (node) => {
     node.callee.name === "Blob"
   );
 };
-export const analyzeNewBlobCall = (node, { onInlineContent }) => {
+export const analyzeNewBlobCall = (
+  node,
+  { onInlineContent, readInlinedFromUrl },
+) => {
   const [firstArg, secondArg] = node.arguments;
   if (!firstArg) {
     return;
@@ -25,7 +28,7 @@ export const analyzeNewBlobCall = (node, { onInlineContent }) => {
   if (firstArg.elements.length !== 1) {
     return;
   }
-  const typePropertyNode = getTypePropertyNode(secondArg);
+  const typePropertyNode = findPropertyNodeByName(secondArg, "type");
   if (!typePropertyNode) {
     return;
   }
@@ -40,6 +43,7 @@ export const analyzeNewBlobCall = (node, { onInlineContent }) => {
     onInlineContent({
       type: "new_blob_first_arg",
       contentType,
+      inlinedFromUrl: readInlinedFromUrl(node),
       start: nodeHoldingContent.start,
       end: nodeHoldingContent.end,
       line: nodeHoldingContent.loc.start.line,

package/src/js/js_static_analysis/new_inline_content.js

@@ -1,6 +1,6 @@
 import {
   isStringLiteralNode,
-  getTypePropertyNode,
+  findPropertyNodeByName,
   extractContentInfo,
 } from "./helpers.js";
 
@@ -11,7 +11,10 @@ export const isNewInlineContentCall = (node) => {
     node.callee.name === "__InlineContent__"
   );
 };
-export const analyzeNewInlineContentCall = (node, { onInlineContent }) => {
+export const analyzeNewInlineContentCall = (
+  node,
+  { onInlineContent, readInlinedFromUrl },
+) => {
   const [firstArg, secondArg] = node.arguments;
   if (!firstArg) {
     return;
@@ -19,7 +22,7 @@ export const analyzeNewInlineContentCall = (node, { onInlineContent }) => {
   if (!secondArg) {
     return;
   }
-  const typePropertyNode = getTypePropertyNode(secondArg);
+  const typePropertyNode = findPropertyNodeByName(secondArg, "type");
   if (!typePropertyNode) {
     return;
   }
@@ -27,6 +30,7 @@ export const analyzeNewInlineContentCall = (node, { onInlineContent }) => {
   if (!isStringLiteralNode(typePropertyValueNode)) {
     return;
   }
+
   const nodeHoldingContent = firstArg;
   const contentType = typePropertyValueNode.value;
   const contentInfo = extractContentInfo(nodeHoldingContent);
@@ -34,6 +38,7 @@ export const analyzeNewInlineContentCall = (node, { onInlineContent }) => {
     onInlineContent({
       type: "new_inline_content_first_arg",
       contentType,
+      inlinedFromUrl: readInlinedFromUrl(node),
       start: nodeHoldingContent.start,
       end: nodeHoldingContent.end,
       line: nodeHoldingContent.loc.start.line,

package/src/js/js_static_analysis/web_worker_entry_point.js

@@ -1,4 +1,4 @@
-import { getTypePropertyNode, isStringLiteralNode } from "./helpers.js";
+import { findPropertyNodeByName, isStringLiteralNode } from "./helpers.js";
 import { isNewUrlCall, analyzeNewUrlCall } from "./new_url.js";
 import {
   isImportMetaResolveCall,
@@ -102,7 +102,7 @@ const analyzeWorkerCallArguments = (
   let typePropertyNode;
   const secondArgNode = node.arguments[1];
   if (secondArgNode) {
-    typePropertyNode = getTypePropertyNode(secondArgNode);
+    typePropertyNode = findPropertyNodeByName(secondArgNode, "type");
     if (typePropertyNode) {
       const typePropertyValueNode = typePropertyNode.value;
       if (isStringLiteralNode(typePropertyValueNode)) {

package/src/js/parse_js_urls.js

@@ -73,6 +73,49 @@ export const parseJsUrls = ({
       ...inlineContentInfo,
     });
   };
+
+  const getCommentBeforeClosingParenthesis = (
+    // either new InlineContent() or JSON.parse() for instance
+    callNode,
+  ) => {
+    const args = callNode.arguments;
+    let commentMustStartAfter;
+    let commentMustEndBefore;
+    if (args.length === 0) {
+      commentMustStartAfter = callNode.start;
+      commentMustEndBefore = callNode.end - 1;
+    } else {
+      const lastArg = args[args.length - 1];
+      commentMustStartAfter = lastArg.start;
+      commentMustEndBefore = callNode.end - 1;
+    }
+    for (const comment of ast.comments) {
+      if (
+        comment.start > commentMustStartAfter &&
+        comment.end < commentMustEndBefore
+      ) {
+        return comment;
+      }
+    }
+    return null;
+  };
+
+  const readInlinedFromUrl = (node) => {
+    let inlinedFromUrl;
+    const commentBeforeClosingParenthesis =
+      getCommentBeforeClosingParenthesis(node);
+    if (commentBeforeClosingParenthesis) {
+      const text = commentBeforeClosingParenthesis.text;
+      const inlinedFromUrlIndex = text.indexOf("inlinedFromUrl=");
+      if (inlinedFromUrlIndex > -1) {
+        inlinedFromUrl = text.slice(
+          inlinedFromUrlIndex + "inlinedFromUrl=".length,
+        );
+      }
+    }
+    return inlinedFromUrl;
+  };
+
   ancestor(ast, {
     ImportDeclaration: (node) => {
       analyzeImportDeclaration(node, { onUrl });
@@ -125,6 +168,7 @@ export const parseJsUrls = ({
       if (inlineContent && isJSONParseCall(node)) {
         analyzeJSONParseCall(node, {
           onInlineContent,
+          readInlinedFromUrl,
         });
         return;
       }
@@ -163,12 +207,14 @@ export const parseJsUrls = ({
       if (inlineContent && isNewInlineContentCall(node)) {
         analyzeNewInlineContentCall(node, {
           onInlineContent,
+          readInlinedFromUrl,
         });
         return;
       }
       if (inlineContent && isNewBlobCall(node)) {
         analyzeNewBlobCall(node, {
           onInlineContent,
+          readInlinedFromUrl,
         });
         return;
       }

package/src/js/parse_js_with_acorn.js

@@ -5,6 +5,7 @@ import { createJsParseError } from "./js_parse_error.js";
 
 export const parseJsWithAcorn = ({ js, url, isJsModule }) => {
   const AcornParser = Parser.extend(importAssertions);
+  const comments = [];
 
   try {
     // https://github.com/acornjs/acorn/tree/master/acorn#interface
@@ -13,7 +14,16 @@ export const parseJsWithAcorn = ({ js, url, isJsModule }) => {
       allowAwaitOutsideFunction: true,
       sourceType: isJsModule ? "module" : "script",
       ecmaVersion: 2022,
+      onComment: (block, text, start, end) => {
+        comments.push({
+          block,
+          text,
+          start,
+          end,
+        });
+      },
     });
+    jsAst.comments = comments;
     return jsAst;
   } catch (e) {
     if (e && e.name === "SyntaxError") {

package/src/main.js

@@ -33,6 +33,7 @@ export {
   getHtmlNodePosition,
   getHtmlNodeAttributePosition,
 } from "./html/html_node_position.js";
+export { getUrlForContentInsideHtml } from "./html/html_inline_content_url.js";
 
 // css
 export { parseCssUrls } from "./css/parse_css_urls.js";
@@ -43,3 +44,6 @@ export { injectJsImport } from "./js/babel_utils.js";
 export { parseJsWithAcorn } from "./js/parse_js_with_acorn.js";
 export { parseJsUrls } from "./js/parse_js_urls.js";
 export { visitJsAstUntil } from "./js/visit_js_ast_until.js";
+export { getUrlForContentInsideJs } from "./js/js_inline_content_url.js";
+
+export { generateUrlForInlineContent } from "./inline_content_url.js";