@shikijs/transformers 2.4.2 → 3.0.0

package/dist/index.d.mts

@@ -53,6 +53,7 @@ interface TransformerMetaWordHighlightOptions {
  * Allow using `/word/` in the code snippet meta to mark highlighted words.
  */
 declare function transformerMetaWordHighlight(options?: TransformerMetaWordHighlightOptions): ShikiTransformer$1;
+declare function findAllSubstringIndexes(str: string, substr: string): number[];
 
 interface TransformerNotationDiffOptions extends MatchAlgorithmOptions {
     /**
@@ -202,4 +203,4 @@ interface ShikiTransformerStyleToClass extends ShikiTransformer$1 {
  */
 declare function transformerStyleToClass(options?: TransformerStyleToClassOptions): ShikiTransformerStyleToClass;
 
-export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };
+export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };

package/dist/index.mjs

@@ -1,5 +1,3 @@
-import { warnDeprecated } from '@shikijs/core';
-
 const matchers = [
   [/^(<!--)(.+)(-->)$/, false],
   [/^(\/\*)(.+)(\*\/)$/, false],
@@ -12,6 +10,42 @@ const matchers = [
 function parseComments(lines, jsx, matchAlgorithm) {
   const out = [];
   for (const line of lines) {
+    if (matchAlgorithm === "v3") {
+      const splittedElements = line.children.flatMap((element, idx) => {
+        if (element.type !== "element")
+          return element;
+        const token = element.children[0];
+        if (token.type !== "text")
+          return element;
+        const isLast = idx === line.children.length - 1;
+        const isComment = matchToken(token.value, isLast);
+        if (!isComment)
+          return element;
+        const rawSplits = token.value.split(/(\s+\/\/)/);
+        if (rawSplits.length <= 1)
+          return element;
+        let splits = [rawSplits[0]];
+        for (let i = 1; i < rawSplits.length; i += 2) {
+          splits.push(rawSplits[i] + (rawSplits[i + 1] || ""));
+        }
+        splits = splits.filter(Boolean);
+        if (splits.length <= 1)
+          return element;
+        return splits.map((split) => {
+          return {
+            ...element,
+            children: [
+              {
+                type: "text",
+                value: split
+              }
+            ]
+          };
+        });
+      });
+      if (splittedElements.length !== line.children.length)
+        line.children = splittedElements;
+    }
     const elements = line.children;
     let start = elements.length - 1;
     if (matchAlgorithm === "v1")
@@ -30,17 +64,20 @@ function parseComments(lines, jsx, matchAlgorithm) {
       if (!match)
         continue;
       if (jsx && !isLast && i !== 0) {
+        const isJsxStyle = isValue(elements[i - 1], "{") && isValue(elements[i + 1], "}");
         out.push({
           info: match,
           line,
           token,
-          isJsxStyle: isValue(elements[i - 1], "{") && isValue(elements[i + 1], "}")
+          isLineCommentOnly: elements.length === 3 && token.children.length === 1,
+          isJsxStyle
         });
       } else {
         out.push({
           info: match,
           line,
           token,
+          isLineCommentOnly: elements.length === 1 && token.children.length === 1,
           isJsxStyle: false
         });
       }
@@ -75,18 +112,16 @@ function matchToken(text, isLast) {
   }
 }
 function v1ClearEndCommentPrefix(text) {
-  const regex = /(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(.*)$/;
-  const result = regex.exec(text);
-  if (result && result[1].trim().length === 0) {
-    return text.slice(0, result.index);
+  const match = text.match(/(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(\s*)$/);
+  if (match && match[1].trim().length === 0) {
+    return text.slice(0, match.index);
   }
   return text;
 }
 
 function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm) {
   if (matchAlgorithm == null) {
-    matchAlgorithm = "v1";
-    warnDeprecated('The default `matchAlgorithm: "v1"` is deprecated and will be removed in the future. Please explicitly set `matchAlgorithm: "v3"` in the transformer options.', 3);
+    matchAlgorithm = "v3";
   }
   return {
     name,
@@ -100,9 +135,8 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
       for (const comment of parsed) {
         if (comment.info[1].length === 0)
           continue;
-        const isLineCommentOnly = comment.line.children.length === (comment.isJsxStyle ? 3 : 1);
         let lineIdx = lines.indexOf(comment.line);
-        if (isLineCommentOnly && matchAlgorithm !== "v1")
+        if (comment.isLineCommentOnly && matchAlgorithm !== "v1")
           lineIdx++;
         let replaced = false;
         comment.info[1] = comment.info[1].replace(regex, (...match) => {
@@ -114,13 +148,12 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
         });
         if (!replaced)
           continue;
-        if (matchAlgorithm === "v1") {
+        if (matchAlgorithm === "v1")
           comment.info[1] = v1ClearEndCommentPrefix(comment.info[1]);
-        }
         const isEmpty = comment.info[1].trim().length === 0;
         if (isEmpty)
           comment.info[1] = "";
-        if (isEmpty && isLineCommentOnly) {
+        if (isEmpty && comment.isLineCommentOnly) {
           linesToRemove.push(comment.line);
         } else if (isEmpty && comment.isJsxStyle) {
           comment.line.children.splice(comment.line.children.indexOf(comment.token) - 1, 3);
@@ -133,8 +166,14 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
           }
         }
       }
-      for (const line of linesToRemove)
-        code.children.splice(code.children.indexOf(line), 1);
+      for (const line of linesToRemove) {
+        const index = code.children.indexOf(line);
+        const nextLine = code.children[index + 1];
+        let removeLength = 1;
+        if (nextLine?.type === "text" && nextLine?.value === "\n")
+          removeLength = 2;
+        code.children.splice(index, removeLength);
+      }
     }
   };
 }
@@ -220,9 +259,16 @@ function transformerMetaWordHighlight(options = {}) {
 }
 function findAllSubstringIndexes(str, substr) {
   const indexes = [];
-  let i = -1;
-  while ((i = str.indexOf(substr, i + 1)) !== -1)
-    indexes.push(i);
+  let cursor = 0;
+  while (true) {
+    const index = str.indexOf(substr, cursor);
+    if (index === -1 || index >= str.length)
+      break;
+    if (index < cursor)
+      break;
+    indexes.push(index);
+    cursor = index + substr.length;
+  }
   return indexes;
 }
 
@@ -618,4 +664,4 @@ function cyrb53(str, seed = 0) {
   return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36).slice(0, 6);
 }
 
-export { createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };
+export { createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/transformers",
   "type": "module",
-  "version": "2.4.2",
+  "version": "3.0.0",
   "description": "Collective of common transformers transformers for Shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -18,10 +18,7 @@
   ],
   "sideEffects": false,
   "exports": {
-    ".": {
-      "types": "./dist/index.d.mts",
-      "default": "./dist/index.mjs"
-    }
+    ".": "./dist/index.mjs"
   },
   "main": "./dist/index.mjs",
   "module": "./dist/index.mjs",
@@ -30,8 +27,8 @@
     "dist"
   ],
   "dependencies": {
-    "@shikijs/core": "2.4.2",
-    "@shikijs/types": "2.4.2"
+    "@shikijs/core": "3.0.0",
+    "@shikijs/types": "3.0.0"
   },
   "scripts": {
     "build": "unbuild",

package/dist/index.d.ts

@@ -1,205 +0,0 @@
-import { ShikiTransformerContext, ShikiTransformer } from '@shikijs/core';
-import { Element } from 'hast';
-import { ShikiTransformer as ShikiTransformer$1 } from '@shikijs/types';
-
-type MatchAlgorithm = 'v1' | 'v3';
-interface MatchAlgorithmOptions {
-    /**
-     * Match algorithm to use
-     *
-     * @see https://shiki.style/packages/transformers#matching-algorithm
-     * @default 'v1'
-     */
-    matchAlgorithm?: MatchAlgorithm;
-}
-declare function createCommentNotationTransformer(name: string, regex: RegExp, onMatch: (this: ShikiTransformerContext, match: string[], line: Element, commentNode: Element, lines: Element[], index: number) => boolean, matchAlgorithm: MatchAlgorithm | undefined): ShikiTransformer;
-
-interface TransformerCompactLineOption {
-    /**
-     * 1-based line number.
-     */
-    line: number;
-    classes?: string[];
-}
-/**
- * Transformer for `shiki`'s legacy `lineOptions`
- */
-declare function transformerCompactLineOptions(lineOptions?: TransformerCompactLineOption[]): ShikiTransformer$1;
-
-declare function parseMetaHighlightString(meta: string): number[] | null;
-interface TransformerMetaHighlightOptions {
-    /**
-     * Class for highlighted lines
-     *
-     * @default 'highlighted'
-     */
-    className?: string;
-}
-/**
- * Allow using `{1,3-5}` in the code snippet meta to mark highlighted lines.
- */
-declare function transformerMetaHighlight(options?: TransformerMetaHighlightOptions): ShikiTransformer$1;
-
-declare function parseMetaHighlightWords(meta: string): string[];
-interface TransformerMetaWordHighlightOptions {
-    /**
-     * Class for highlighted words
-     *
-     * @default 'highlighted-word'
-     */
-    className?: string;
-}
-/**
- * Allow using `/word/` in the code snippet meta to mark highlighted words.
- */
-declare function transformerMetaWordHighlight(options?: TransformerMetaWordHighlightOptions): ShikiTransformer$1;
-
-interface TransformerNotationDiffOptions extends MatchAlgorithmOptions {
-    /**
-     * Class for added lines
-     */
-    classLineAdd?: string;
-    /**
-     * Class for removed lines
-     */
-    classLineRemove?: string;
-    /**
-     * Class added to the <pre> element when the current code has diff
-     */
-    classActivePre?: string;
-}
-/**
- * Use `[!code ++]` and `[!code --]` to mark added and removed lines.
- */
-declare function transformerNotationDiff(options?: TransformerNotationDiffOptions): ShikiTransformer$1;
-
-interface TransformerNotationErrorLevelOptions extends MatchAlgorithmOptions {
-    classMap?: Record<string, string | string[]>;
-    /**
-     * Class added to the <pre> element when the current code has diff
-     */
-    classActivePre?: string;
-}
-/**
- * Allow using `[!code error]` `[!code warning]` notation in code to mark highlighted lines.
- */
-declare function transformerNotationErrorLevel(options?: TransformerNotationErrorLevelOptions): ShikiTransformer$1;
-
-interface TransformerNotationFocusOptions extends MatchAlgorithmOptions {
-    /**
-     * Class for focused lines
-     */
-    classActiveLine?: string;
-    /**
-     * Class added to the root element when the code has focused lines
-     */
-    classActivePre?: string;
-}
-/**
- * Allow using `[!code focus]` notation in code to mark focused lines.
- */
-declare function transformerNotationFocus(options?: TransformerNotationFocusOptions): ShikiTransformer$1;
-
-interface TransformerNotationHighlightOptions extends MatchAlgorithmOptions {
-    /**
-     * Class for highlighted lines
-     */
-    classActiveLine?: string;
-    /**
-     * Class added to the root element when the code has highlighted lines
-     */
-    classActivePre?: string;
-}
-/**
- * Allow using `[!code highlight]` notation in code to mark highlighted lines.
- */
-declare function transformerNotationHighlight(options?: TransformerNotationHighlightOptions): ShikiTransformer$1;
-
-interface TransformerNotationWordHighlightOptions extends MatchAlgorithmOptions {
-    /**
-     * Class for highlighted words
-     */
-    classActiveWord?: string;
-    /**
-     * Class added to the root element when the code has highlighted words
-     */
-    classActivePre?: string;
-}
-declare function transformerNotationWordHighlight(options?: TransformerNotationWordHighlightOptions): ShikiTransformer$1;
-
-interface TransformerNotationMapOptions extends MatchAlgorithmOptions {
-    classMap?: Record<string, string | string[]>;
-    /**
-     * Class added to the <pre> element when the current code has diff
-     */
-    classActivePre?: string;
-}
-declare function transformerNotationMap(options?: TransformerNotationMapOptions, name?: string): ShikiTransformer$1;
-
-/**
- * Remove line breaks between lines.
- * Useful when you override `display: block` to `.line` in CSS.
- */
-declare function transformerRemoveLineBreak(): ShikiTransformer$1;
-
-/**
- * Remove notation escapes.
- * Useful when you want to write `// [!code` in markdown.
- * If you process `// [\!code ...]` expression, you can get `// [!code ...]` in the output.
- */
-declare function transformerRemoveNotationEscape(): ShikiTransformer$1;
-
-interface TransformerRenderWhitespaceOptions {
-    /**
-     * Class for tab
-     *
-     * @default 'tab'
-     */
-    classTab?: string;
-    /**
-     * Class for space
-     *
-     * @default 'space'
-     */
-    classSpace?: string;
-    /**
-     * Position of rendered whitespace
-     * @default all position
-     */
-    position?: 'all' | 'boundary' | 'trailing';
-}
-/**
- * Render whitespaces as separate tokens.
- * Apply with CSS, it can be used to render tabs and spaces visually.
- */
-declare function transformerRenderWhitespace(options?: TransformerRenderWhitespaceOptions): ShikiTransformer$1;
-
-interface TransformerStyleToClassOptions {
-    /**
-     * Prefix for class names.
-     * @default '__shiki_'
-     */
-    classPrefix?: string;
-    /**
-     * Suffix for class names.
-     * @default ''
-     */
-    classSuffix?: string;
-    /**
-     * Callback to replace class names.
-     * @default (className) => className
-     */
-    classReplacer?: (className: string) => string;
-}
-interface ShikiTransformerStyleToClass extends ShikiTransformer$1 {
-    getClassRegistry: () => Map<string, Record<string, string> | string>;
-    getCSS: () => string;
-    clearRegistry: () => void;
-}
-/**
- * Remove line breaks between lines.
- * Useful when you override `display: block` to `.line` in CSS.
- */
-declare function transformerStyleToClass(options?: TransformerStyleToClassOptions): ShikiTransformerStyleToClass;
-
-export { type ShikiTransformerStyleToClass, type TransformerCompactLineOption, type TransformerMetaHighlightOptions, type TransformerMetaWordHighlightOptions, type TransformerNotationDiffOptions, type TransformerNotationErrorLevelOptions, type TransformerNotationFocusOptions, type TransformerNotationHighlightOptions, type TransformerNotationMapOptions, type TransformerNotationWordHighlightOptions, type TransformerRenderWhitespaceOptions, type TransformerStyleToClassOptions, createCommentNotationTransformer, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderWhitespace, transformerStyleToClass };