@shikijs/core 1.16.1 → 1.16.2

package/dist/chunk-engines.d.mts

@@ -49,6 +49,12 @@ interface JavaScriptRegexEngineOptions {
      * Cache for regex patterns.
      */
     cache?: Map<string, RegExp | Error>;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-js` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
 }
 
 export type { Awaitable as A, JavaScriptRegexEngineOptions as J, LoadWasmOptions as L, MaybeArray as M, OnigurumaLoadOptions as O, PatternScanner as P, RequireKeys as R, StringLiteralUnion as S, WebAssemblyInstantiator as W, RegexEngine as a, MaybeGetter as b, MaybeModule as c, RegexEngineString as d, WebAssemblyInstance as e, LoadWasmOptionsPlain as f };

package/dist/index.d.mts

@@ -131,6 +131,10 @@ declare function loadWasm(options: LoadWasmOptions): Promise<void>;
 
 declare function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise<RegexEngine>;
 
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
 /**
  * Use the modern JavaScript RegExp engine to implement the OnigScanner.
  *
@@ -470,4 +474,4 @@ declare class ShikiError extends Error {
     constructor(message: string);
 }
 
-export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, CreatedBundledHighlighterOptions, GrammarState, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, JavaScriptRegexEngineOptions, LanguageInput, LoadWasmOptions, MaybeArray, MaybeGetter, PlainTextLanguage, Position, RegexEngine, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };
+export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, CreatedBundledHighlighterOptions, GrammarState, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, JavaScriptRegexEngineOptions, LanguageInput, LoadWasmOptions, MaybeArray, MaybeGetter, PlainTextLanguage, Position, RegexEngine, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };

package/dist/index.mjs

@@ -106,7 +106,8 @@ function splitToken(token, offsets) {
  * Split 2D tokens array by given breakpoints.
  */
 function splitTokens(tokens, breakpoints) {
-    const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
+    const sorted = Array
+        .from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
         .sort((a, b) => a - b);
     if (!sorted.length)
         return tokens;
@@ -752,7 +753,8 @@ function dimColor(color) {
     if (hexMatch) {
         if (hexMatch[3]) {
             // convert from #rrggbbaa to #rrggbb(aa/2)
-            const alpha = Math.round(Number.parseInt(hexMatch[3], 16) / 2)
+            const alpha = Math
+                .round(Number.parseInt(hexMatch[3], 16) / 2)
                 .toString(16)
                 .padStart(2, '0');
             return `#${hexMatch[1]}${hexMatch[2]}${alpha}`;
@@ -763,7 +765,8 @@ function dimColor(color) {
         }
         else {
             // convert from #rgb to #rrggbb80
-            return `#${Array.from(hexMatch[1])
+            return `#${Array
+                .from(hexMatch[1])
                 .map(x => `${x}${x}`)
                 .join('')}80`;
         }
@@ -964,7 +967,8 @@ function explainThemeScope(themeSettingsSelectors, scope, parentScopes) {
  * Get tokens with multiple themes
  */
 function codeToTokensWithThemes(internal, code, options) {
-    const themes = Object.entries(options.themes)
+    const themes = Object
+        .entries(options.themes)
         .filter(i => i[1])
         .map(i => ({ color: i[0], theme: i[1] }));
     const tokens = syncThemesTokenization(...themes.map(t => codeToTokensBase(internal, code, {
@@ -1051,7 +1055,8 @@ function codeToTokens(internal, code, options) {
     let rootStyle;
     if ('themes' in options) {
         const { defaultColor = 'light', cssVariablePrefix = '--shiki-', } = options;
-        const themes = Object.entries(options.themes)
+        const themes = Object
+            .entries(options.themes)
             .filter(i => i[1])
             .map(i => ({ color: i[0], theme: i[1] }))
             .sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0);
@@ -5862,6 +5867,8 @@ class RegExpConversionError extends SyntaxError {
   }
 }
 
+const UNNECESSARY_ESCAPE_CHAR_CLASS = new Set("!?:=+$(){}_><# ");
+const UNNECESSARY_ESCAPE = new Set("-!:=_>< ");
 const TABLE_POSIX = {
   alnum: "0-9A-Za-z",
   alpha: "A-Za-z",
@@ -5878,13 +5885,23 @@ const TABLE_POSIX = {
   xdigit: "0-9A-Fa-f",
   word: "\\w"
 };
+const TABLE_SLASH_P = {
+  alnum: "0-9A-Za-z",
+  alpha: "A-Za-z",
+  alphabetic: "A-Za-z",
+  blank: "\\s",
+  greek: "\\p{Script=Greek}",
+  print: "\\p{L}\\p{N}\\p{P}\\p{S}\\p{Zs}",
+  word: "\\w"
+};
 const KNOWN_FLAGS = /* @__PURE__ */ new Set("gimsuyx");
 function syntaxLowering(input, options = {}) {
   const {
     preserveFlags = false,
     removePossessiveQuantifier = false,
     removeAtomicGroup = false,
-    convertHexDigitsShorthand = false
+    convertHexDigitsShorthand = false,
+    convertUnicodeCategory = false
   } = options;
   let output = "";
   const flags = /* @__PURE__ */ new Set();
@@ -5899,7 +5916,7 @@ function syntaxLowering(input, options = {}) {
         freeSpacingLocal.shift();
       }
       const head = stack[0];
-      const wsEscape = freeSpacingGlobal || freeSpacingLocal.length;
+      const freeSpacing = freeSpacingGlobal || freeSpacingLocal.length;
       if (char === "\\") {
         if (convertHexDigitsShorthand) {
           if (input[i + 1] === "h") {
@@ -5925,11 +5942,41 @@ function syntaxLowering(input, options = {}) {
             continue;
           }
         }
+        if (convertUnicodeCategory && input[i + 1] === "p" && input[i + 2] === "{") {
+          const end = input.indexOf("}", i + 3);
+          if (end === -1) {
+            throw new RegExpConversionError(
+              "Unmatched \\p{...}",
+              { pattern: input, converted: output, cursor: i }
+            );
+          }
+          const name = input.slice(i + 3, end);
+          const resolved = TABLE_SLASH_P[name.toLowerCase()];
+          if (resolved) {
+            if (head === "[") {
+              output += resolved;
+            } else {
+              output += `[${resolved}]`;
+            }
+            i = end + 1;
+            continue;
+          }
+        }
+        if (head === "[" && UNNECESSARY_ESCAPE_CHAR_CLASS.has(input[i + 1])) {
+          output += input[i + 1];
+          i += 2;
+          continue;
+        }
+        if (head !== "[" && UNNECESSARY_ESCAPE.has(input[i + 1])) {
+          output += input[i + 1];
+          i += 2;
+          continue;
+        }
         output += char + input[i + 1];
         i += 2;
         continue;
       }
-      if (char === "#" && wsEscape && input[i - 1].match(/\s/) && head !== "[") {
+      if (char === "#" && freeSpacing && input[i - 1].match(/\s/) && head !== "[") {
         for (let j = i + 1; j <= input.length; j++) {
           if (input[j] === "\n" || j === input.length) {
             i = j;
@@ -6068,7 +6115,7 @@ function syntaxLowering(input, options = {}) {
           continue;
         }
       }
-      if (!(wsEscape && head !== "[" && char.match(/\s/))) {
+      if (!(freeSpacing && head !== "[" && char.match(/\s/))) {
         output += char;
       }
       i += 1;
@@ -6108,7 +6155,9 @@ function construct(pattern, options = {}) {
   if (options.ignoreContiguousAnchors) {
     pattern = pattern.replace(/\\G/g, "");
   }
-  pattern = pattern.replace(/\\A/g, "^").replace(/\\x\{([^}]*)\}/g, (m, hex) => `\\u${hex.padStart(4, "0")}`).replace(/\(\?(-)?(\w+):/g, (_, neg, flagStr) => {
+  if (pattern.includes("\\p{"))
+    flagSet.add("u");
+  pattern = pattern.replace(/\\A/g, "^").replace(/\\Z/gi, "$").replace(/\\x\{([^}]*)\}/g, (m, hex) => `\\u${hex.padStart(4, "0")}`).replace(/\(\?(-)?(\w+):/g, (_, neg, flagStr) => {
     if (neg) {
       for (const flag of flagStr)
         flagSet.delete(flag);
@@ -6155,6 +6204,7 @@ function onigurumaToRegexp(pattern, options = {}) {
     removePossessiveQuantifier: true,
     removeAtomicGroup: true,
     convertHexDigitsShorthand: true,
+    convertUnicodeCategory: true,
     ...options
   });
   return construct(converted, {
@@ -6165,15 +6215,30 @@ function onigurumaToRegexp(pattern, options = {}) {
 }
 
 const MAX = 4294967295;
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+function defaultJavaScriptRegexConstructor(pattern) {
+    return onigurumaToRegexp(pattern
+        .replace(/\|\\G(\||\))/g, '$1')
+        .replace(/(\(|\|)\\G\|/g, '$1')
+        // YAML specific handling; TODO: move to tm-grammars
+        .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'), {
+        flags: 'dgm',
+        ignoreContiguousAnchors: true,
+    });
+}
 class JavaScriptScanner {
     patterns;
     cache;
     forgiving;
+    regexConstructor;
     regexps;
-    constructor(patterns, cache, forgiving) {
+    constructor(patterns, cache, forgiving, regexConstructor = defaultJavaScriptRegexConstructor) {
         this.patterns = patterns;
         this.cache = cache;
         this.forgiving = forgiving;
+        this.regexConstructor = regexConstructor;
         this.regexps = patterns.map((p) => {
             const cached = cache?.get(p);
             if (cached) {
@@ -6185,14 +6250,7 @@ class JavaScriptScanner {
                 throw cached;
             }
             try {
-                const regex = onigurumaToRegexp(p
-                    .replace(/\|\\G(\||\))/g, '$1')
-                    .replace(/(\(|\|)\\G\|/g, '$1')
-                    // YAML specific handling; TODO: move to tm-grammars
-                    .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'), {
-                    flags: 'dgm',
-                    ignoreContiguousAnchors: true,
-                });
+                const regex = regexConstructor(p);
                 cache?.set(p, regex);
                 return regex;
             }
@@ -6276,7 +6334,7 @@ function createJavaScriptRegexEngine(options = {}) {
     const { forgiving = false, cache = new Map(), } = options;
     return {
         createScanner(patterns) {
-            return new JavaScriptScanner(patterns, cache, forgiving);
+            return new JavaScriptScanner(patterns, cache, forgiving, options.regexConstructor);
         },
         createString(s) {
             return {
@@ -6286,4 +6344,4 @@ function createJavaScriptRegexEngine(options = {}) {
     };
 }
 
-export { FontStyle, ShikiError, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };
+export { FontStyle, ShikiError, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.16.1",
+  "version": "1.16.2",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -65,7 +65,7 @@
   },
   "devDependencies": {
     "hast-util-to-html": "^9.0.2",
-    "oniguruma-to-js": "^0.3.0",
+    "oniguruma-to-js": "^0.3.1",
     "vscode-oniguruma": "^1.7.0"
   },
   "scripts": {