@shikijs/core 4.0.1 → 4.1.0

package/dist/index.mjs

@@ -2,10 +2,9 @@ import { ShikiError as ShikiError$1 } from "@shikijs/types";
 import { ShikiError, applyColorReplacements, codeToTokensBase as codeToTokensBase$1, codeToTokensWithThemes, codeToTokensWithThemes as codeToTokensWithThemes$1, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitive as createShikiPrimitive$1, createShikiPrimitiveAsync, createShikiPrimitiveAsync as createShikiPrimitiveAsync$1, getLastGrammarState, getLastGrammarStateFromMap, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, normalizeGetter, normalizeTheme, resolveColorReplacements, setLastGrammarStateToMap, splitLines, splitLines as splitLines$1, toArray, tokenizeWithTheme } from "@shikijs/primitive";
 import { FontStyle } from "@shikijs/vscode-textmate";
 import { toHtml } from "hast-util-to-html";
-
-export * from "@shikijs/types"
-
+export * from "@shikijs/types";
 //#region src/utils/hast.ts
+const RE_WHITESPACE = /\s+/g;
 /**
 * Utility to append class to a hast node
 *
@@ -15,15 +14,18 @@ function addClassToHast(node, className) {
 	if (!className) return node;
 	node.properties ||= {};
 	node.properties.class ||= [];
-	if (typeof node.properties.class === "string") node.properties.class = node.properties.class.split(/\s+/g);
+	if (typeof node.properties.class === "string") node.properties.class = node.properties.class.split(RE_WHITESPACE);
 	if (!Array.isArray(node.properties.class)) node.properties.class = [];
-	const targets = Array.isArray(className) ? className : className.split(/\s+/g);
+	const targets = Array.isArray(className) ? className : className.split(RE_WHITESPACE);
 	for (const c of targets) if (c && !node.properties.class.includes(c)) node.properties.class.push(c);
 	return node;
 }
-
 //#endregion
 //#region src/utils/strings.ts
+const RE_LANG_ATTR = /:?lang=["']([^"']+)["']/g;
+const RE_CODE_FENCE = /(?:```|~~~)([\w-]+)/g;
+const RE_LATEX_BEGIN = /\\begin\{([\w-]+)\}/g;
+const RE_SCRIPT_LANG = /<script\s+(?:type|lang)=["']([^"']+)["']/gi;
 /**
 * Creates a converter between index and position in a code block.
 *
@@ -34,7 +36,7 @@ function createPositionConverter(code) {
 	function indexToPos(index) {
 		if (index === code.length) return {
 			line: lines.length - 1,
-			character: lines[lines.length - 1].length
+			character: lines.at(-1).length
 		};
 		let character = index;
 		let line = 0;
@@ -81,33 +83,28 @@ function createPositionConverter(code) {
 */
 function guessEmbeddedLanguages(code, _lang, highlighter) {
 	const langs = /* @__PURE__ */ new Set();
-	for (const match of code.matchAll(/:?lang=["']([^"']+)["']/g)) {
+	for (const match of code.matchAll(RE_LANG_ATTR)) {
 		const lang = match[1].toLowerCase().trim();
 		if (lang) langs.add(lang);
 	}
-	for (const match of code.matchAll(/(?:```|~~~)([\w-]+)/g)) {
+	for (const match of code.matchAll(RE_CODE_FENCE)) {
 		const lang = match[1].toLowerCase().trim();
 		if (lang) langs.add(lang);
 	}
-	for (const match of code.matchAll(/\\begin\{([\w-]+)\}/g)) {
+	for (const match of code.matchAll(RE_LATEX_BEGIN)) {
 		const lang = match[1].toLowerCase().trim();
 		if (lang) langs.add(lang);
 	}
-	for (const match of code.matchAll(/<script\s+(?:type|lang)=["']([^"']+)["']/gi)) {
+	for (const match of code.matchAll(RE_SCRIPT_LANG)) {
 		const fullType = match[1].toLowerCase().trim();
 		const lang = fullType.includes("/") ? fullType.split("/").pop() : fullType;
 		if (lang) langs.add(lang);
 	}
-	if (!highlighter) return Array.from(langs);
+	if (!highlighter) return [...langs];
 	const bundle = highlighter.getBundledLanguages();
-	return Array.from(langs).filter((l) => l && bundle[l]);
+	return [...langs].filter((l) => l && bundle[l]);
 }
-
-//#endregion
-//#region src/utils/constants.ts
-const DEFAULT_COLOR_LIGHT_DARK = "light-dark()";
 const COLOR_KEYS = ["color", "background-color"];
-
 //#endregion
 //#region src/utils/tokens.ts
 /**
@@ -137,7 +134,7 @@ function splitToken(token, offsets) {
 * Split 2D tokens array by given breakpoints.
 */
 function splitTokens(tokens, breakpoints) {
-	const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints)).sort((a, b) => a - b);
+	const sorted = [...breakpoints instanceof Set ? breakpoints : new Set(breakpoints)].sort((a, b) => a - b);
 	if (!sorted.length) return tokens;
 	return tokens.map((line) => {
 		return line.flatMap((token) => {
@@ -163,7 +160,7 @@ function flatTokenVariants(merged, variantsOrder, cssVariablePrefix, defaultColo
 	styles.forEach((cur, idx) => {
 		for (const key of styleKeys) {
 			const value = cur[key] || "inherit";
-			if (idx === 0 && defaultColor && COLOR_KEYS.includes(key)) if (defaultColor === DEFAULT_COLOR_LIGHT_DARK && styles.length > 1) {
+			if (idx === 0 && defaultColor && COLOR_KEYS.includes(key)) if (defaultColor === "light-dark()" && styles.length > 1) {
 				const lightIndex = variantsOrder.findIndex((t) => t === "light");
 				const darkIndex = variantsOrder.findIndex((t) => t === "dark");
 				if (lightIndex === -1 || darkIndex === -1) throw new ShikiError$1("When using `defaultColor: \"light-dark()\"`, you must provide both `light` and `dark` themes");
@@ -194,7 +191,6 @@ function stringifyTokenStyle(token) {
 	if (typeof token === "string") return token;
 	return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(";");
 }
-
 //#endregion
 //#region src/transformer-decorations.ts
 /**
@@ -248,7 +244,7 @@ function transformerDecorations() {
 		code(codeEl) {
 			if (!this.options.decorations?.length) return;
 			const ctx = getContext(this);
-			const lines = Array.from(codeEl.children).filter((i) => i.type === "element" && i.tagName === "span");
+			const lines = [...codeEl.children].filter((i) => i.type === "element" && i.tagName === "span");
 			if (lines.length !== ctx.converter.lines.length) throw new ShikiError$1(`Number of lines in code element (${lines.length}) does not match the number of lines in the source (${ctx.converter.lines.length}). Failed to apply decorations.`);
 			function applyLineSection(line, start, end, decoration) {
 				const lineEl = lines[line];
@@ -335,7 +331,6 @@ function stringify(el) {
 	if (el.type === "element") return el.children.map(stringify).join("");
 	return "";
 }
-
 //#endregion
 //#region src/highlight/_get-transformers.ts
 const builtInTransformers = [/* @__PURE__ */ transformerDecorations()];
@@ -367,7 +362,6 @@ function sortTransformersByEnforcement(transformers) {
 		normal
 	};
 }
-
 //#endregion
 //#region ../../node_modules/.pnpm/ansi-sequence-parser@1.1.3/node_modules/ansi-sequence-parser/dist/index.js
 var namedColors = [
@@ -603,9 +597,10 @@ function createColorPalette(namedColorsMap = defaultNamedColorsMap) {
 	}
 	return { value };
 }
-
 //#endregion
 //#region src/highlight/code-to-tokens-ansi.ts
+const RE_HEX_COLOR = /#([0-9a-f]{3,8})/i;
+const RE_CSS_VAR_ANSI = /var\((--[\w-]+-ansi-[\w-]+)\)/;
 /**
 * Default ANSI palette (VSCode compatible fallbacks)
 * Used when the theme does not define terminal.ansi* colors.
@@ -667,7 +662,7 @@ function tokenizeAnsiWithTheme(theme, fileContents, options) {
 * Adds 50% alpha to a hex color string or the "-dim" postfix to a CSS variable
 */
 function dimColor(color) {
-	const hexMatch = color.match(/#([0-9a-f]{3,8})/i);
+	const hexMatch = color.match(RE_HEX_COLOR);
 	if (hexMatch) {
 		const hex = hexMatch[1];
 		if (hex.length === 8) {
@@ -687,11 +682,10 @@ function dimColor(color) {
 			return `#${r}${r}${g}${g}${b}${b}80`;
 		}
 	}
-	const cssVarMatch = color.match(/var\((--[\w-]+-ansi-[\w-]+)\)/);
+	const cssVarMatch = color.match(RE_CSS_VAR_ANSI);
 	if (cssVarMatch) return `var(${cssVarMatch[1]}-dim)`;
 	return color;
 }
-
 //#endregion
 //#region src/highlight/code-to-tokens-base.ts
 /**
@@ -707,7 +701,6 @@ function codeToTokensBase(primitive, code, options = {}) {
 	}
 	return codeToTokensBase$1(primitive, code, options);
 }
-
 //#endregion
 //#region src/highlight/code-to-tokens.ts
 /**
@@ -729,9 +722,9 @@ function codeToTokens(primitive, code, options) {
 			theme: i[1]
 		})).sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0);
 		if (themes.length === 0) throw new ShikiError$1("`themes` option must not be empty");
-		const themeTokens = codeToTokensWithThemes$1(primitive, code, options);
+		const themeTokens = codeToTokensWithThemes$1(primitive, code, options, codeToTokensBase);
 		grammarState = getLastGrammarStateFromMap(themeTokens);
-		if (defaultColor && DEFAULT_COLOR_LIGHT_DARK !== defaultColor && !themes.find((t) => t.color === defaultColor)) throw new ShikiError$1(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``);
+		if (defaultColor && "light-dark()" !== defaultColor && !themes.some((t) => t.color === defaultColor)) throw new ShikiError$1(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``);
 		const themeRegs = themes.map((t) => primitive.getTheme(t.theme));
 		const themesOrder = themes.map((t) => t.color);
 		tokens = themeTokens.map((line) => line.map((token) => flatTokenVariants(token, themesOrder, cssVariablePrefix, defaultColor, colorsRendering)));
@@ -764,7 +757,7 @@ function mapThemeColors(themes, themeRegs, themeColorReplacements, cssVariablePr
 		const value = applyColorReplacements(themeRegs[idx][property], themeColorReplacements[idx]) || "inherit";
 		const cssVar = `${cssVariablePrefix + t.color}${property === "bg" ? "-bg" : ""}:${value}`;
 		if (idx === 0 && defaultColor) {
-			if (defaultColor === DEFAULT_COLOR_LIGHT_DARK && themes.length > 1) {
+			if (defaultColor === "light-dark()" && themes.length > 1) {
 				const lightIndex = themes.findIndex((t) => t.color === "light");
 				const darkIndex = themes.findIndex((t) => t.color === "dark");
 				if (lightIndex === -1 || darkIndex === -1) throw new ShikiError$1("When using `defaultColor: \"light-dark()\"`, you must provide both `light` and `dark` themes");
@@ -776,9 +769,10 @@ function mapThemeColors(themes, themeRegs, themeColorReplacements, cssVariablePr
 		return null;
 	}).filter((i) => !!i).join(";");
 }
-
 //#endregion
 //#region src/highlight/code-to-hast.ts
+const RE_WHITESPACE_ONLY = /^\s+$/;
+const RE_LEADING_TRAILING_WHITESPACE = /^(\s*)(.*?)(\s*)$/;
 function codeToHast(primitive, code, options, transformerContext = {
 	meta: {},
 	options,
@@ -957,7 +951,7 @@ function mergeWhitespaceTokens(tokens) {
 		let firstOffset;
 		line.forEach((token, idx) => {
 			const couldMerge = !(token.fontStyle && (token.fontStyle & FontStyle.Underline || token.fontStyle & FontStyle.Strikethrough));
-			if (couldMerge && token.content.match(/^\s+$/) && line[idx + 1]) {
+			if (couldMerge && RE_WHITESPACE_ONLY.test(token.content) && line[idx + 1]) {
 				if (firstOffset === void 0) firstOffset = token.offset;
 				carryOnContent += token.content;
 			} else if (carryOnContent) {
@@ -980,8 +974,8 @@ function mergeWhitespaceTokens(tokens) {
 function splitWhitespaceTokens(tokens) {
 	return tokens.map((line) => {
 		return line.flatMap((token) => {
-			if (token.content.match(/^\s+$/)) return token;
-			const match = token.content.match(/^(\s*)(.*?)(\s*)$/);
+			if (RE_WHITESPACE_ONLY.test(token.content)) return token;
+			const match = token.content.match(RE_LEADING_TRAILING_WHITESPACE);
 			if (!match) return token;
 			const [, leading, content, trailing] = match;
 			if (!leading && !trailing) return token;
@@ -1010,7 +1004,7 @@ function mergeAdjacentStyledTokens(tokens) {
 				newLine.push({ ...token });
 				continue;
 			}
-			const prevToken = newLine[newLine.length - 1];
+			const prevToken = newLine.at(-1);
 			const prevStyle = stringifyTokenStyle(prevToken.htmlStyle || getTokenStyleObject(prevToken));
 			const currentStyle = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token));
 			const isPrevDecorated = prevToken.fontStyle && (prevToken.fontStyle & FontStyle.Underline || prevToken.fontStyle & FontStyle.Strikethrough);
@@ -1021,7 +1015,6 @@ function mergeAdjacentStyledTokens(tokens) {
 		return newLine;
 	});
 }
-
 //#endregion
 //#region src/highlight/code-to-html.ts
 const hastToHtml = toHtml;
@@ -1039,7 +1032,6 @@ function codeToHtml(primitive, code, options) {
 	for (const transformer of getTransformers(options)) result = transformer.postprocess?.call(context, result, options) || result;
 	return result;
 }
-
 //#endregion
 //#region src/constructors/highlighter.ts
 /**
@@ -1105,7 +1097,6 @@ function makeSingletonHighlighterCore(createHighlighter) {
 	return getSingletonHighlighterCore;
 }
 const getSingletonHighlighterCore = /* @__PURE__ */ makeSingletonHighlighterCore(createHighlighterCore);
-
 //#endregion
 //#region src/constructors/bundle-factory.ts
 function createBundledHighlighter(options) {
@@ -1216,7 +1207,6 @@ function createSingletonShorthands(createHighlighter, config) {
 		}
 	};
 }
-
 //#endregion
 //#region src/theme-css-variables.ts
 /**
@@ -1412,6 +1402,5 @@ function createCssVariablesTheme(options = {}) {
 	});
 	return theme;
 }
-
 //#endregion
-export { ShikiError, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createBundledHighlighter, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createPositionConverter, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitiveAsync, createSingletonShorthands, flatTokenVariants, getLastGrammarState, getSingletonHighlighterCore, getTokenStyleObject, guessEmbeddedLanguages, hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };
+export { ShikiError, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createBundledHighlighter, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createPositionConverter, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitiveAsync, createSingletonShorthands, flatTokenVariants, getLastGrammarState, getSingletonHighlighterCore, getTokenStyleObject, guessEmbeddedLanguages, hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };

package/dist/textmate.mjs

@@ -1,3 +1,2 @@
-export * from "@shikijs/vscode-textmate"
-
-export {  };
+export * from "@shikijs/vscode-textmate";
+export {};

package/dist/types.mjs

@@ -1,3 +1,2 @@
-export * from "@shikijs/types"
-
-export {  };
+export * from "@shikijs/types";
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "4.0.1",
+  "version": "4.1.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -49,8 +49,8 @@
     "@shikijs/vscode-textmate": "^10.0.2",
     "@types/hast": "^3.0.4",
     "hast-util-to-html": "^9.0.5",
-    "@shikijs/primitive": "4.0.1",
-    "@shikijs/types": "4.0.1"
+    "@shikijs/primitive": "4.1.0",
+    "@shikijs/types": "4.1.0"
   },
   "scripts": {
     "build": "tsdown",