@shikijs/transformers 4.0.2 → 4.2.0

package/dist/index.mjs

@@ -1,4 +1,7 @@
 //#region src/shared/parse-comments.ts
+const RE_SPLIT_COMMENT = /(\s+\/\/)/;
+const RE_V1_END_COMMENT_PREFIX = /(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(\s*)$/;
+const RE_V3_END_COMMENT_PREFIX = /(?:\/\/|#|;{1,2}|%{1,2}|--)(\s*)$/;
 /**
 * some comment formats have to be located at the end of line
 * hence we can skip matching them for other tokens
@@ -24,7 +27,7 @@ function parseComments(lines, jsx, matchAlgorithm) {
 				if (token.type !== "text") return element;
 				const isLast = idx === line.children.length - 1;
 				if (!matchToken(token.value, isLast)) return element;
-				const rawSplits = token.value.split(/(\s+\/\/)/);
+				const rawSplits = token.value.split(RE_SPLIT_COMMENT);
 				if (rawSplits.length <= 1) return element;
 				let splits = [rawSplits[0]];
 				for (let i = 1; i < rawSplits.length; i += 2) splits.push(rawSplits[i] + (rawSplits[i + 1] || ""));
@@ -130,15 +133,24 @@ function matchToken(text, isLast) {
 * For matchAlgorithm v1
 */
 function v1ClearEndCommentPrefix(text) {
-	const match = text.match(/(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(\s*)$/);
+	const match = text.match(RE_V1_END_COMMENT_PREFIX);
 	if (match && match[1].trim().length === 0) return text.slice(0, match.index);
 	return text;
 }
-
+/**
+* Remove empty comment prefixes at line end, e.g. `// `
+*
+* For matchAlgorithm v3
+*/
+function v3ClearEndCommentPrefix(text) {
+	const match = text.match(RE_V3_END_COMMENT_PREFIX);
+	if (match && match[1].trim().length === 0) return text.slice(0, match.index).trimEnd();
+	return text;
+}
 //#endregion
 //#region src/shared/notation-transformer.ts
 function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm) {
-	if (matchAlgorithm == null) matchAlgorithm = "v3";
+	matchAlgorithm ??= "v3";
 	return {
 		name,
 		code(code) {
@@ -162,6 +174,7 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
 				});
 				if (!replaced) continue;
 				if (matchAlgorithm === "v1") comment.info[1] = v1ClearEndCommentPrefix(comment.info[1]);
+				else if (matchAlgorithm === "v3") comment.info[1] = v3ClearEndCommentPrefix(comment.info[1]);
 				const isEmpty = comment.info[1].trim().length === 0;
 				if (isEmpty) comment.info[1] = "";
 				if (isEmpty && comment.isLineCommentOnly) linesToRemove.push(comment.line);
@@ -194,7 +207,6 @@ function createCommentNotationTransformer(name, regex, onMatch, matchAlgorithm)
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/compact-line-options.ts
 /**
@@ -210,12 +222,12 @@ function transformerCompactLineOptions(lineOptions = []) {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/meta-highlight.ts
+const RE_HIGHLIGHT_LINES = /\{([\d,-]+)\}/;
 function parseMetaHighlightString(meta) {
 	if (!meta) return null;
-	const match = meta.match(/\{([\d,-]+)\}/);
+	const match = meta.match(RE_HIGHLIGHT_LINES);
 	if (!match) return null;
 	return match[1].split(",").flatMap((v) => {
 		const range = v.split("-").map((n) => Number.parseInt(n, 10));
@@ -241,12 +253,13 @@ function transformerMetaHighlight(options = {}) {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/meta-highlight-word.ts
+const RE_WORD_MATCH = /\/((?:\\.|[^/])+)\//g;
+const RE_ESCAPE_BACKSLASH = /\\(.)/g;
 function parseMetaHighlightWords(meta) {
 	if (!meta) return [];
-	return Array.from(meta.matchAll(/\/((?:\\.|[^/])+)\//g)).map((v) => v[1].replace(/\\(.)/g, "$1"));
+	return [...meta.matchAll(RE_WORD_MATCH)].map((v) => v[1].replace(RE_ESCAPE_BACKSLASH, "$1"));
 }
 /**
 * Allow using `/word/` in the code snippet meta to mark highlighted words.
@@ -282,11 +295,11 @@ function findAllSubstringIndexes(str, substr) {
 	}
 	return indexes;
 }
-
 //#endregion
 //#region src/transformers/notation-map.ts
+const RE_ESCAPE_SPECIAL = /[.*+?^${}()|[\]\\]/g;
 function escapeRegExp(str) {
-	return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return str.replace(RE_ESCAPE_SPECIAL, "\\$&");
 }
 function transformerNotationMap(options = {}, name = "@shikijs/transformers:notation-map") {
 	const { classMap = {}, classActivePre = void 0, classActiveCode = void 0 } = options;
@@ -298,7 +311,6 @@ function transformerNotationMap(options = {}, name = "@shikijs/transformers:nota
 		return true;
 	}, options.matchAlgorithm);
 }
-
 //#endregion
 //#region src/transformers/notation-diff.ts
 /**
@@ -316,7 +328,6 @@ function transformerNotationDiff(options = {}) {
 		matchAlgorithm: options.matchAlgorithm
 	}, "@shikijs/transformers:notation-diff");
 }
-
 //#endregion
 //#region src/transformers/notation-error-level.ts
 /**
@@ -335,7 +346,6 @@ function transformerNotationErrorLevel(options = {}) {
 		matchAlgorithm: options.matchAlgorithm
 	}, "@shikijs/transformers:notation-error-level");
 }
-
 //#endregion
 //#region src/transformers/notation-focus.ts
 /**
@@ -350,7 +360,6 @@ function transformerNotationFocus(options = {}) {
 		matchAlgorithm: options.matchAlgorithm
 	}, "@shikijs/transformers:notation-focus");
 }
-
 //#endregion
 //#region src/transformers/notation-highlight.ts
 /**
@@ -368,7 +377,6 @@ function transformerNotationHighlight(options = {}) {
 		matchAlgorithm: options.matchAlgorithm
 	}, "@shikijs/transformers:notation-highlight");
 }
-
 //#endregion
 //#region src/shared/highlight-word.ts
 function highlightWordInLine(line, ignoredElement, word, className) {
@@ -433,20 +441,20 @@ function inheritElement(original, overrides) {
 		...overrides
 	};
 }
-
 //#endregion
 //#region src/transformers/notation-highlight-word.ts
+const RE_CODE_WORD = /\s*\[!code word:((?:\\.|[^:\]])+)(:\d+)?\]/;
+const RE_UNESCAPE = /\\(.)/g;
 function transformerNotationWordHighlight(options = {}) {
 	const { classActiveWord = "highlighted-word", classActivePre = void 0 } = options;
-	return createCommentNotationTransformer("@shikijs/transformers:notation-highlight-word", /\s*\[!code word:((?:\\.|[^:\]])+)(:\d+)?\]/, function([_, word, range], _line, comment, lines, index) {
+	return createCommentNotationTransformer("@shikijs/transformers:notation-highlight-word", RE_CODE_WORD, function([_, word, range], _line, comment, lines, index) {
 		const lineNum = range ? Number.parseInt(range.slice(1), 10) : lines.length;
-		word = word.replace(/\\(.)/g, "$1");
+		word = word.replace(RE_UNESCAPE, "$1");
 		for (let i = index; i < Math.min(index + lineNum, lines.length); i++) highlightWordInLine.call(this, lines[i], comment, word, classActiveWord);
 		if (classActivePre) this.addClassToHast(this.pre, classActivePre);
 		return true;
 	}, options.matchAlgorithm);
 }
-
 //#endregion
 //#region src/transformers/remove-comments.ts
 /**
@@ -475,7 +483,6 @@ function transformerRemoveComments(options = {}) {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/remove-line-breaks.ts
 /**
@@ -490,7 +497,6 @@ function transformerRemoveLineBreak() {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/remove-notation-escape.ts
 /**
@@ -511,9 +517,10 @@ function transformerRemoveNotationEscape() {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/render-indent-guides.ts
+const RE_INDENT_META = /\{indent:(\d+|false)\}/;
+const RE_NON_BLANK = /[^ \t]/;
 /**
 * Render indentations as separate tokens.
 * Apply with CSS, it can be used to render indent guides visually.
@@ -522,7 +529,7 @@ function transformerRenderIndentGuides(options = {}) {
 	return {
 		name: "@shikijs/transformers:render-indent-guides",
 		code(hast) {
-			const indent = Number(this.options.meta?.indent ?? this.options.meta?.__raw?.match(/\{indent:(\d+|false)\}/)?.[1] ?? options.indent ?? 2);
+			const indent = Number(this.options.meta?.indent ?? this.options.meta?.__raw?.match(RE_INDENT_META)?.[1] ?? options.indent ?? 2);
 			if (Number.isNaN(indent) || indent <= 0) return hast;
 			const indentRegex = new RegExp(` {${indent}}| {0,${indent - 1}}\t| {1,}$`, "g");
 			const emptyLines = [];
@@ -535,7 +542,7 @@ function transformerRenderIndentGuides(options = {}) {
 					continue;
 				}
 				const text = first.children[0];
-				const blanks = text.value.split(/[^ \t]/, 1)[0];
+				const blanks = text.value.split(RE_NON_BLANK, 1)[0];
 				const ranges = [];
 				for (const match of blanks.matchAll(indentRegex)) {
 					const start = match.index;
@@ -570,7 +577,6 @@ function transformerRenderIndentGuides(options = {}) {
 		}
 	};
 }
-
 //#endregion
 //#region src/shared/utils.ts
 function isTab(part) {
@@ -613,9 +619,9 @@ function splitSpaces(parts, type, renderContinuousSpaces = true) {
 		...parts.slice(parts.length - rightCount)
 	];
 }
-
 //#endregion
 //#region src/transformers/render-whitespace.ts
+const RE_SPACE_OR_TAB = /([ \t])/;
 /**
 * Render whitespaces as separate tokens.
 * Apply with CSS, it can be used to render tabs and spaces visually.
@@ -643,7 +649,7 @@ function transformerRenderWhitespace(options = {}) {
 					if (position === "leading" && index !== 0) return token;
 					const node = token.children[0];
 					if (node.type !== "text" || !node.value) return token;
-					const parts = splitSpaces(node.value.split(/([ \t])/).filter((i) => i.length), position === "boundary" && index === last && last !== 0 ? "trailing" : position, position !== "trailing" && position !== "leading");
+					const parts = splitSpaces(node.value.split(RE_SPACE_OR_TAB).filter((i) => i.length), position === "boundary" && index === last && last !== 0 ? "trailing" : position, position !== "trailing" && position !== "leading");
 					if (parts.length <= 1) return token;
 					return parts.map((part) => {
 						const clone = {
@@ -665,7 +671,6 @@ function transformerRenderWhitespace(options = {}) {
 		}
 	};
 }
-
 //#endregion
 //#region src/transformers/style-to-class.ts
 /**
@@ -734,6 +739,5 @@ function cyrb53(str, seed = 0) {
 	h2 ^= Math.imul(h1 ^ h1 >>> 13, 3266489909);
 	return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(36).slice(0, 6);
 }
-
 //#endregion
-export { createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveComments, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderIndentGuides, transformerRenderWhitespace, transformerStyleToClass };
+export { createCommentNotationTransformer, findAllSubstringIndexes, parseMetaHighlightString, parseMetaHighlightWords, transformerCompactLineOptions, transformerMetaHighlight, transformerMetaWordHighlight, transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationMap, transformerNotationWordHighlight, transformerRemoveComments, transformerRemoveLineBreak, transformerRemoveNotationEscape, transformerRenderIndentGuides, transformerRenderWhitespace, transformerStyleToClass };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/transformers",
   "type": "module",
-  "version": "4.0.2",
+  "version": "4.2.0",
   "description": "Collective of common transformers transformers for Shiki",
   "author": "Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -30,8 +30,8 @@
     "node": ">=20"
   },
   "dependencies": {
-    "@shikijs/core": "4.0.2",
-    "@shikijs/types": "4.0.2"
+    "@shikijs/core": "4.2.0",
+    "@shikijs/types": "4.2.0"
   },
   "scripts": {
     "build": "tsdown",