@shikijs/transformers 2.4.1 → 2.5.0

package/dist/index.d.mts

@@ -53,6 +53,7 @@ interface TransformerMetaWordHighlightOptions {
  * Allow using `/word/` in the code snippet meta to mark highlighted words.
  */
 declare function transformerMetaWordHighlight(options?: TransformerMetaWordHighlightOptions): ShikiTransformer$1;
+declare function findAllSubstringIndexes(str: string, substr: string): number[];
 
 interface TransformerNotationDiffOptions extends MatchAlgorithmOptions {
     /**
@@ -202,4 +203,4 @@ interface ShikiTransformerStyleToClass extends ShikiTransformer$1 {
  */
 declare function transformerStyleToClass(options?: TransformerStyleToClassOptions): ShikiTransformerStyleToClass;
 
-export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };
+export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };

package/dist/index.d.ts

@@ -53,6 +53,7 @@ interface TransformerMetaWordHighlightOptions {
  * Allow using `/word/` in the code snippet meta to mark highlighted words.
  */
 declare function transformerMetaWordHighlight(options?: TransformerMetaWordHighlightOptions): ShikiTransformer$1;
+declare function findAllSubstringIndexes(str: string, substr: string): number[];
 
 interface TransformerNotationDiffOptions extends MatchAlgorithmOptions {
     /**
@@ -202,4 +203,4 @@ interface ShikiTransformerStyleToClass extends ShikiTransformer$1 {
  */
 declare function transformerStyleToClass(options?: TransformerStyleToClassOptions): ShikiTransformerStyleToClass;
 
-export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };
+export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };

package/dist/index.mjs

@@ -12,6 +12,42 @@ const matchers = [
 function parseComments(lines, jsx, matchAlgorithm) {
   const out = [];
   for (const line of lines) {
+    if (matchAlgorithm === "v3") {
+      const splittedElements = line.children.flatMap((element, idx) => {
+        if (element.type !== "element")
+          return element;
+        const token = element.children[0];
+        if (token.type !== "text")
+          return element;
+        const isLast = idx === line.children.length - 1;
+        const isComment = matchToken(token.value, isLast);
+        if (!isComment)
+          return element;
+        const rawSplits = token.value.split(/(\s+\/\/)/);
+        if (rawSplits.length <= 1)
+          return element;
+        let splits = [rawSplits[0]];
+        for (let i = 1; i < rawSplits.length; i += 2) {
+          splits.push(rawSplits[i] + (rawSplits[i + 1] || ""));
+        }
+        splits = splits.filter(Boolean);
+        if (splits.length <= 1)
+          return element;
+        return splits.map((split) => {
+          return {
+            ...element,
+            children: [
+              {
+                type: "text",
+                value: split
+              }
+            ]
+          };
+        });
+      });
+      if (splittedElements.length !== line.children.length)
+        line.children = splittedElements;
+    }
     const elements = line.children;
     let start = elements.length - 1;
     if (matchAlgorithm === "v1")
@@ -30,17 +66,20 @@ function parseComments(lines, jsx, matchAlgorithm) {
       if (!match)
         continue;
       if (jsx && !isLast && i !== 0) {
+        const isJsxStyle = isValue(elements[i - 1], "{") && isValue(elements[i + 1], "}");
         out.push({
           info: match,
           line,
           token,
-          isJsxStyle: isValue(elements[i - 1], "{") && isValue(elements[i + 1], "}")
+          isLineCommentOnly: elements.length === 3 && token.children.length === 1,
+          isJsxStyle
         });
       } else {
         out.push({
           info: match,
           line,
           token,
+          isLineCommentOnly: elements.length === 1 && token.children.length === 1,
           isJsxStyle: false
         });
       }
@@ -75,10 +114,9 @@ function matchToken(text, isLast) {
   }
 }
 function v1ClearEndCommentPrefix(text) {
-  const regex = /(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(.*)$/;
-  const result = regex.exec(text);
-  if (result && result[1].trim().length === 0) {
-    return text.slice(0, result.index);
+  const match = text.match(/(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(\s*)$/);
+  if (match && match[1].trim().length === 0) {
+    return text.slice(0, match.index);
   }
   return text;
 }
@@ -100,9 +138,8 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
       for (const comment of parsed) {
         if (comment.info[1].length === 0)
           continue;
-        const isLineCommentOnly = comment.line.children.length === (comment.isJsxStyle ? 3 : 1);
         let lineIdx = lines.indexOf(comment.line);
-        if (isLineCommentOnly && matchAlgorithm !== "v1")
+        if (comment.isLineCommentOnly && matchAlgorithm !== "v1")
           lineIdx++;
         let replaced = false;
         comment.info[1] = comment.info[1].replace(regex, (...match) => {
@@ -114,13 +151,12 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
         });
         if (!replaced)
           continue;
-        if (matchAlgorithm === "v1") {
+        if (matchAlgorithm === "v1")
           comment.info[1] = v1ClearEndCommentPrefix(comment.info[1]);
-        }
         const isEmpty = comment.info[1].trim().length === 0;
         if (isEmpty)
           comment.info[1] = "";
-        if (isEmpty && isLineCommentOnly) {
+        if (isEmpty && comment.isLineCommentOnly) {
           linesToRemove.push(comment.line);
         } else if (isEmpty && comment.isJsxStyle) {
           comment.line.children.splice(comment.line.children.indexOf(comment.token) - 1, 3);
@@ -133,8 +169,14 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
           }
         }
       }
-      for (const line of linesToRemove)
-        code.children.splice(code.children.indexOf(line), 1);
+      for (const line of linesToRemove) {
+        const index = code.children.indexOf(line);
+        const nextLine = code.children[index + 1];
+        let removeLength = 1;
+        if (nextLine?.type === "text" && nextLine?.value === "\n")
+          removeLength = 2;
+        code.children.splice(index, removeLength);
+      }
     }
   };
 }
@@ -220,9 +262,16 @@ function transformerMetaWordHighlight(options = {}) {
 }
 function findAllSubstringIndexes(str, substr) {
   const indexes = [];
-  let i = -1;
-  while ((i = str.indexOf(substr, i + 1)) !== -1)
-    indexes.push(i);
+  let cursor = 0;
+  while (true) {
+    const index = str.indexOf(substr, cursor);
+    if (index === -1 || index >= str.length)
+      break;
+    if (index < cursor)
+      break;
+    indexes.push(index);
+    cursor = index + substr.length;
+  }
   return indexes;
 }
 
@@ -429,6 +478,12 @@ function transformerRemoveNotationEscape() {
   };
 }
 
+function isTab(part) {
+  return part === "	";
+}
+function isSpace(part) {
+  return part === " " || part === "	";
+}
 function separateContinuousSpaces(inputs) {
   const result = [];
   let current = "";
@@ -451,12 +506,6 @@ function separateContinuousSpaces(inputs) {
   bump();
   return result;
 }
-function isTab(part) {
-  return part === "	";
-}
-function isSpace(part) {
-  return part === " " || part === "	";
-}
 function splitSpaces(parts, type, renderContinuousSpaces = true) {
   if (type === "all")
     return parts;
@@ -618,4 +667,4 @@ function cyrb53(str, seed = 0) {
   return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36).slice(0, 6);
 }
 
-export { createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };
+export { createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/transformers",
   "type": "module",
-  "version": "2.4.1",
+  "version": "2.5.0",
   "description": "Collective of common transformers transformers for Shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -30,8 +30,8 @@
     "dist"
   ],
   "dependencies": {
-    "@shikijs/types": "2.4.1",
-    "@shikijs/core": "2.4.1"
+    "@shikijs/core": "2.5.0",
+    "@shikijs/types": "2.5.0"
   },
   "scripts": {
     "build": "unbuild",