@shikijs/core 2.3.2 → 2.4.0

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
+import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, Awaitable, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
 export * from '@shikijs/types';
 import { Root, Element } from 'hast';
 import { JavaScriptRegexEngineOptions } from '@shikijs/engine-javascript';
@@ -79,7 +79,13 @@ interface ShorthandsBundle<L extends string, T extends string> {
     getLastGrammarState: ((element: ThemedToken[][] | Root) => GrammarState) | ((code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>);
 }
 declare function makeSingletonHighlighter<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
-declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
+interface CreateSingletonShorthandsOptions<L extends string, T extends string> {
+    /**
+     * A custom function to guess embedded languages to be loaded.
+     */
+    guessEmbeddedLanguages?: (code: string, lang: string | undefined, highlighter: HighlighterGeneric<L, T>) => Awaitable<string[] | undefined>;
+}
+declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>, config?: CreateSingletonShorthandsOptions<L, T>): ShorthandsBundle<L, T>;
 
 /**
  * Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -292,4 +298,4 @@ declare function enableDeprecationWarnings(emitDeprecation?: DeprecationTarget |
  */
 declare function warnDeprecated(message: string, version?: DeprecationTarget): void;
 
-export { type CssVariablesThemeOptions, type ShorthandsBundle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createOnigurumaEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, enableDeprecationWarnings, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations, warnDeprecated };
+export { type CreateSingletonShorthandsOptions, type CssVariablesThemeOptions, type ShorthandsBundle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createOnigurumaEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, enableDeprecationWarnings, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations, warnDeprecated };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
+import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, Awaitable, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
 export * from '@shikijs/types';
 import { Root, Element } from 'hast';
 import { JavaScriptRegexEngineOptions } from '@shikijs/engine-javascript';
@@ -79,7 +79,13 @@ interface ShorthandsBundle<L extends string, T extends string> {
     getLastGrammarState: ((element: ThemedToken[][] | Root) => GrammarState) | ((code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>);
 }
 declare function makeSingletonHighlighter<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
-declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
+interface CreateSingletonShorthandsOptions<L extends string, T extends string> {
+    /**
+     * A custom function to guess embedded languages to be loaded.
+     */
+    guessEmbeddedLanguages?: (code: string, lang: string | undefined, highlighter: HighlighterGeneric<L, T>) => Awaitable<string[] | undefined>;
+}
+declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>, config?: CreateSingletonShorthandsOptions<L, T>): ShorthandsBundle<L, T>;
 
 /**
  * Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -292,4 +298,4 @@ declare function enableDeprecationWarnings(emitDeprecation?: DeprecationTarget |
  */
 declare function warnDeprecated(message: string, version?: DeprecationTarget): void;
 
-export { type CssVariablesThemeOptions, type ShorthandsBundle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createOnigurumaEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, enableDeprecationWarnings, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations, warnDeprecated };
+export { type CreateSingletonShorthandsOptions, type CssVariablesThemeOptions, type ShorthandsBundle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createOnigurumaEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, enableDeprecationWarnings, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations, warnDeprecated };

package/dist/index.mjs

@@ -284,7 +284,7 @@ function transformerDecorations() {
           };
         } else {
           const line = converter.lines[p.line];
-          if (line === undefined)
+          if (line === void 0)
             throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Lines length: ${converter.lines.length}`);
           if (p.character < 0 || p.character > line.length)
             throw new ShikiError(`Invalid decoration position ${JSON.stringify(p)}. Line ${p.line} length: ${line.length}`);
@@ -466,52 +466,52 @@ var decorations = {
   3: "italic",
   4: "underline",
   7: "reverse",
+  8: "hidden",
   9: "strikethrough"
 };
 
 // src/parser.ts
 function findSequence(value, position) {
-  const nextEscape = value.indexOf("\x1B[", position);
+  const nextEscape = value.indexOf("\x1B", position);
   if (nextEscape !== -1) {
-    const nextClose = value.indexOf("m", nextEscape);
-    return {
-      sequence: value.substring(nextEscape + 2, nextClose).split(";"),
-      startPosition: nextEscape,
-      position: nextClose + 1
-    };
+    if (value[nextEscape + 1] === "[") {
+      const nextClose = value.indexOf("m", nextEscape);
+      if (nextClose !== -1) {
+        return {
+          sequence: value.substring(nextEscape + 2, nextClose).split(";"),
+          startPosition: nextEscape,
+          position: nextClose + 1
+        };
+      }
+    }
   }
   return {
     position: value.length
   };
 }
-function parseColor(sequence, index) {
-  let offset = 1;
-  const colorMode = sequence[index + offset++];
-  let color;
+function parseColor(sequence) {
+  const colorMode = sequence.shift();
   if (colorMode === "2") {
-    const rgb = [
-      sequence[index + offset++],
-      sequence[index + offset++],
-      sequence[index + offset]
-    ].map((x) => Number.parseInt(x));
-    if (rgb.length === 3 && !rgb.some((x) => Number.isNaN(x))) {
-      color = {
-        type: "rgb",
-        rgb
-      };
-    }
+    const rgb = sequence.splice(0, 3).map((x) => Number.parseInt(x));
+    if (rgb.length !== 3 || rgb.some((x) => Number.isNaN(x)))
+      return;
+    return {
+      type: "rgb",
+      rgb
+    };
   } else if (colorMode === "5") {
-    const colorIndex = Number.parseInt(sequence[index + offset]);
-    if (!Number.isNaN(colorIndex)) {
-      color = { type: "table", index: Number(colorIndex) };
+    const index = sequence.shift();
+    if (index) {
+      return { type: "table", index: Number(index) };
     }
   }
-  return [offset, color];
 }
 function parseSequence(sequence) {
   const commands = [];
-  for (let i = 0; i < sequence.length; i++) {
-    const code = sequence[i];
+  while (sequence.length > 0) {
+    const code = sequence.shift();
+    if (!code)
+      continue;
     const codeInt = Number.parseInt(code);
     if (Number.isNaN(codeInt))
       continue;
@@ -532,6 +532,12 @@ function parseSequence(sequence) {
           type: "resetDecoration",
           value: decoration
         });
+        if (decoration === "dim") {
+          commands.push({
+            type: "resetDecoration",
+            value: "bold"
+          });
+        }
       }
     } else if (codeInt <= 37) {
       commands.push({
@@ -539,14 +545,13 @@ function parseSequence(sequence) {
         value: { type: "named", name: namedColors[codeInt - 30] }
       });
     } else if (codeInt === 38) {
-      const [offset, color] = parseColor(sequence, i);
+      const color = parseColor(sequence);
       if (color) {
         commands.push({
           type: "setForegroundColor",
           value: color
         });
       }
-      i += offset;
     } else if (codeInt === 39) {
       commands.push({
         type: "resetForegroundColor"
@@ -557,18 +562,27 @@ function parseSequence(sequence) {
         value: { type: "named", name: namedColors[codeInt - 40] }
       });
     } else if (codeInt === 48) {
-      const [offset, color] = parseColor(sequence, i);
+      const color = parseColor(sequence);
       if (color) {
         commands.push({
           type: "setBackgroundColor",
           value: color
         });
       }
-      i += offset;
     } else if (codeInt === 49) {
       commands.push({
         type: "resetBackgroundColor"
       });
+    } else if (codeInt === 53) {
+      commands.push({
+        type: "setDecoration",
+        value: "overline"
+      });
+    } else if (codeInt === 55) {
+      commands.push({
+        type: "resetDecoration",
+        value: "overline"
+      });
     } else if (codeInt >= 90 && codeInt <= 97) {
       commands.push({
         type: "setForegroundColor",
@@ -722,7 +736,7 @@ function tokenizeAnsiWithTheme(theme, fileContents, options) {
         bgColor = token.foreground ? colorPalette.value(token.foreground) : theme.fg;
       } else {
         color = token.foreground ? colorPalette.value(token.foreground) : theme.fg;
-        bgColor = token.background ? colorPalette.value(token.background) : undefined;
+        bgColor = token.background ? colorPalette.value(token.background) : void 0;
       }
       color = applyColorReplacements(color, colorReplacements);
       bgColor = applyColorReplacements(bgColor, colorReplacements);
@@ -830,8 +844,8 @@ function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     colorMap,
     {
       ...options,
-      grammarState: undefined,
-      grammarContextCode: undefined
+      grammarState: void 0,
+      grammarContextCode: void 0
     }
   ).stateStack : INITIAL;
   let actual = [];
@@ -857,7 +871,7 @@ function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     let tokensWithScopes;
     let tokensWithScopesIndex;
     if (options.includeExplanation) {
-      resultWithScopes = grammar.tokenizeLine(line, stateStack);
+      resultWithScopes = grammar.tokenizeLine(line, stateStack, tokenizeTimeLimit);
       tokensWithScopes = resultWithScopes.tokens;
       tokensWithScopesIndex = 0;
     }
@@ -1020,7 +1034,7 @@ function codeToTokensWithThemes(internal, code, options) {
   const mergedGrammarState = themedTokens[0].state ? new GrammarState(
     Object.fromEntries(themedTokens.map((s) => [s.theme, s.state?.getInternalStack(s.theme)])),
     themedTokens[0].state.lang
-  ) : undefined;
+  ) : void 0;
   if (mergedGrammarState)
     setLastGrammarStateToMap(mergedTokens, mergedGrammarState);
   return mergedTokens;
@@ -1091,7 +1105,7 @@ function codeToTokens(internal, code, options) {
     fg = themes.map((t, idx) => (idx === 0 && defaultColor ? "" : `${cssVariablePrefix + t.color}:`) + (applyColorReplacements(themeRegs[idx].fg, themeColorReplacements[idx]) || "inherit")).join(";");
     bg = themes.map((t, idx) => (idx === 0 && defaultColor ? "" : `${cssVariablePrefix + t.color}-bg:`) + (applyColorReplacements(themeRegs[idx].bg, themeColorReplacements[idx]) || "inherit")).join(";");
     themeName = `shiki-themes ${themeRegs.map((t) => t.name).join(" ")}`;
-    rootStyle = defaultColor ? undefined : [fg, bg].join(";");
+    rootStyle = defaultColor ? void 0 : [fg, bg].join(";");
   } else if ("theme" in options) {
     const colorReplacements = resolveColorReplacements(options.theme, options);
     tokens = codeToTokensBase(
@@ -1408,7 +1422,7 @@ function normalizeTheme(rawTheme) {
   theme.settings ||= [];
   let { bg, fg } = theme;
   if (!bg || !fg) {
-    const globalSetting = theme.settings ? theme.settings.find((s) => !s.name && !s.scope) : undefined;
+    const globalSetting = theme.settings ? theme.settings.find((s) => !s.name && !s.scope) : void 0;
     if (globalSetting?.settings?.foreground)
       fg = globalSetting.settings.foreground;
     if (globalSetting?.settings?.background)
@@ -1808,6 +1822,8 @@ async function createHighlighterCore(options) {
     codeToTokens: (code, options2) => codeToTokens(internal, code, options2),
     codeToHast: (code, options2) => codeToHast(internal, code, options2),
     codeToHtml: (code, options2) => codeToHtml(internal, code, options2),
+    getBundledLanguages: () => ({}),
+    getBundledThemes: () => ({}),
     ...internal,
     getInternalContext: () => internal
   };
@@ -1821,6 +1837,8 @@ function createHighlighterCoreSync(options) {
     codeToTokens: (code, options2) => codeToTokens(internal, code, options2),
     codeToHast: (code, options2) => codeToHast(internal, code, options2),
     codeToHtml: (code, options2) => codeToHtml(internal, code, options2),
+    getBundledLanguages: () => ({}),
+    getBundledThemes: () => ({}),
     ...internal,
     getInternalContext: () => internal
   };
@@ -1905,6 +1923,12 @@ function createdBundledHighlighter(arg1, arg2, arg3) {
       },
       loadTheme(...themes) {
         return core.loadTheme(...themes.map(resolveTheme));
+      },
+      getBundledLanguages() {
+        return bundledLanguages;
+      },
+      getBundledThemes() {
+        return bundledThemes;
       }
     };
   }
@@ -1931,45 +1955,41 @@ function makeSingletonHighlighter(createHighlighter) {
   }
   return getSingletonHighlighter;
 }
-function createSingletonShorthands(createHighlighter) {
+function createSingletonShorthands(createHighlighter, config) {
   const getSingletonHighlighter = makeSingletonHighlighter(createHighlighter);
+  async function get(code, options) {
+    const shiki = await getSingletonHighlighter({
+      langs: [options.lang],
+      themes: "theme" in options ? [options.theme] : Object.values(options.themes)
+    });
+    const langs = await config?.guessEmbeddedLanguages?.(code, options.lang, shiki);
+    if (langs) {
+      await shiki.loadLanguage(...langs);
+    }
+    return shiki;
+  }
   return {
     getSingletonHighlighter(options) {
       return getSingletonHighlighter(options);
     },
     async codeToHtml(code, options) {
-      const shiki = await getSingletonHighlighter({
-        langs: [options.lang],
-        themes: "theme" in options ? [options.theme] : Object.values(options.themes)
-      });
+      const shiki = await get(code, options);
       return shiki.codeToHtml(code, options);
     },
     async codeToHast(code, options) {
-      const shiki = await getSingletonHighlighter({
-        langs: [options.lang],
-        themes: "theme" in options ? [options.theme] : Object.values(options.themes)
-      });
+      const shiki = await get(code, options);
       return shiki.codeToHast(code, options);
     },
     async codeToTokens(code, options) {
-      const shiki = await getSingletonHighlighter({
-        langs: [options.lang],
-        themes: "theme" in options ? [options.theme] : Object.values(options.themes)
-      });
+      const shiki = await get(code, options);
       return shiki.codeToTokens(code, options);
     },
     async codeToTokensBase(code, options) {
-      const shiki = await getSingletonHighlighter({
-        langs: [options.lang],
-        themes: [options.theme]
-      });
+      const shiki = await get(code, options);
       return shiki.codeToTokensBase(code, options);
     },
     async codeToTokensWithThemes(code, options) {
-      const shiki = await getSingletonHighlighter({
-        langs: [options.lang],
-        themes: Object.values(options.themes).filter(Boolean)
-      });
+      const shiki = await get(code, options);
       return shiki.codeToTokensWithThemes(code, options);
     },
     async getLastGrammarState(code, options) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "2.3.2",
+  "version": "2.4.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -60,12 +60,12 @@
     "dist"
   ],
   "dependencies": {
-    "@shikijs/vscode-textmate": "^10.0.1",
+    "@shikijs/vscode-textmate": "^10.0.2",
     "@types/hast": "^3.0.4",
     "hast-util-to-html": "^9.0.4",
-    "@shikijs/engine-javascript": "2.3.2",
-    "@shikijs/types": "2.3.2",
-    "@shikijs/engine-oniguruma": "2.3.2"
+    "@shikijs/engine-javascript": "2.4.0",
+    "@shikijs/types": "2.4.0",
+    "@shikijs/engine-oniguruma": "2.4.0"
   },
   "scripts": {
     "build": "unbuild",