@shikijs/core 1.21.0 → 1.22.0

package/dist/index.d.mts

@@ -1,8 +1,7 @@
-import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
+import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
 export * from '@shikijs/types';
 import { Root, Element } from 'hast';
 import { JavaScriptRegexEngineOptions } from '@shikijs/engine-javascript';
-import { IGrammar } from '@shikijs/vscode-textmate';
 export { FontStyle, EncodedTokenMetadata as StackElementMetadata } from '@shikijs/vscode-textmate';
 export { toHtml as hastToHtml } from 'hast-util-to-html';
 
@@ -77,7 +76,7 @@ interface ShorthandsBundle<L extends string, T extends string> {
      * Shorthand for `getLastGrammarState` with auto-loaded theme and language.
      * A singleton highlighter it maintained internally.
      */
-    getLastGrammarState: (code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>;
+    getLastGrammarState: ((element: ThemedToken[][] | Root) => GrammarState) | ((code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>);
 }
 declare function makeSingletonHighlighter<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
 declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
@@ -144,7 +143,7 @@ declare function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise
 declare function loadWasm(options: LoadWasmOptions): Promise<void>;
 
 declare function codeToHast(internal: ShikiInternal, code: string, options: CodeToHastOptions, transformerContext?: ShikiTransformerContextCommon): Root;
-declare function tokensToHast(tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextSource): Root;
+declare function tokensToHast(tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextSource, grammarState?: GrammarState | undefined): Root;
 
 /**
  * Get highlighted code in HTML.
@@ -164,7 +163,7 @@ declare function tokenizeAnsiWithTheme(theme: ThemeRegistrationResolved, fileCon
  * Code to tokens, with a simple theme.
  */
 declare function codeToTokensBase(internal: ShikiInternal, code: string, options?: CodeToTokensBaseOptions): ThemedToken[][];
-declare function tokenizeWithTheme(code: string, grammar: IGrammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions): ThemedToken[][];
+declare function tokenizeWithTheme(code: string, grammar: Grammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions): ThemedToken[][];
 
 /**
  * Get tokens with multiple themes

package/dist/index.d.ts

@@ -1,8 +1,7 @@
-import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
+import { CreatedBundledHighlighterOptions, CreateHighlighterFactory, LanguageInput, ThemeInput, HighlighterCoreOptions, CodeToHastOptions, CodeToTokensOptions, TokensResult, RequireKeys, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, BundledHighlighterOptions, HighlighterGeneric, GrammarState, HighlighterCore, ShikiInternal, RegexEngine, LoadWasmOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, ThemeRegistrationResolved, TokenizeWithThemeOptions, Grammar, ThemeRegistrationAny, ThemeRegistration, ShikiTransformer, MaybeArray, PlainTextLanguage, SpecialLanguage, SpecialTheme, MaybeGetter, TokenStyles, Position } from '@shikijs/types';
 export * from '@shikijs/types';
 import { Root, Element } from 'hast';
 import { JavaScriptRegexEngineOptions } from '@shikijs/engine-javascript';
-import { IGrammar } from '@shikijs/vscode-textmate';
 export { FontStyle, EncodedTokenMetadata as StackElementMetadata } from '@shikijs/vscode-textmate';
 export { toHtml as hastToHtml } from 'hast-util-to-html';
 
@@ -77,7 +76,7 @@ interface ShorthandsBundle<L extends string, T extends string> {
      * Shorthand for `getLastGrammarState` with auto-loaded theme and language.
      * A singleton highlighter it maintained internally.
      */
-    getLastGrammarState: (code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>;
+    getLastGrammarState: ((element: ThemedToken[][] | Root) => GrammarState) | ((code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>);
 }
 declare function makeSingletonHighlighter<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
 declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
@@ -144,7 +143,7 @@ declare function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise
 declare function loadWasm(options: LoadWasmOptions): Promise<void>;
 
 declare function codeToHast(internal: ShikiInternal, code: string, options: CodeToHastOptions, transformerContext?: ShikiTransformerContextCommon): Root;
-declare function tokensToHast(tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextSource): Root;
+declare function tokensToHast(tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextSource, grammarState?: GrammarState | undefined): Root;
 
 /**
  * Get highlighted code in HTML.
@@ -164,7 +163,7 @@ declare function tokenizeAnsiWithTheme(theme: ThemeRegistrationResolved, fileCon
  * Code to tokens, with a simple theme.
  */
 declare function codeToTokensBase(internal: ShikiInternal, code: string, options?: CodeToTokensBaseOptions): ThemedToken[][];
-declare function tokenizeWithTheme(code: string, grammar: IGrammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions): ThemedToken[][];
+declare function tokenizeWithTheme(code: string, grammar: Grammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions): ThemedToken[][];
 
 /**
  * Get tokens with multiple themes

package/dist/index.mjs

@@ -181,6 +181,101 @@ class ShikiError extends Error {
   }
 }
 
+var __defProp$2 = Object.defineProperty;
+var __defNormalProp$2 = (obj, key, value) => key in obj ? __defProp$2(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField$2 = (obj, key, value) => {
+  __defNormalProp$2(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+const _grammarStateMap = /* @__PURE__ */ new WeakMap();
+function setLastGrammarStateToMap(keys, state) {
+  _grammarStateMap.set(keys, state);
+}
+function getLastGrammarStateFromMap(keys) {
+  return _grammarStateMap.get(keys);
+}
+class GrammarState {
+  constructor(...args) {
+    /**
+     * Theme to Stack mapping
+     */
+    __publicField$2(this, "_stacks", {});
+    __publicField$2(this, "lang");
+    if (args.length === 2) {
+      const [stacksMap, lang] = args;
+      this.lang = lang;
+      this._stacks = stacksMap;
+    } else {
+      const [stack, lang, theme] = args;
+      this.lang = lang;
+      this._stacks = { [theme]: stack };
+    }
+  }
+  get themes() {
+    return Object.keys(this._stacks);
+  }
+  get theme() {
+    return this.themes[0];
+  }
+  get _stack() {
+    return this._stacks[this.theme];
+  }
+  /**
+   * Static method to create a initial grammar state.
+   */
+  static initial(lang, themes) {
+    return new GrammarState(
+      Object.fromEntries(toArray(themes).map((theme) => [theme, INITIAL])),
+      lang
+    );
+  }
+  /**
+   * Get the internal stack object.
+   * @internal
+   */
+  getInternalStack(theme = this.theme) {
+    return this._stacks[theme];
+  }
+  /**
+   * @deprecated use `getScopes` instead
+   */
+  get scopes() {
+    return getScopes(this._stacks[this.theme]);
+  }
+  getScopes(theme = this.theme) {
+    return getScopes(this._stacks[theme]);
+  }
+  toJSON() {
+    return {
+      lang: this.lang,
+      theme: this.theme,
+      themes: this.themes,
+      scopes: this.scopes
+    };
+  }
+}
+function getScopes(stack) {
+  const scopes = [];
+  const visited = /* @__PURE__ */ new Set();
+  function pushScope(stack2) {
+    if (visited.has(stack2))
+      return;
+    visited.add(stack2);
+    const name = stack2?.nameScopesList?.scopeName;
+    if (name)
+      scopes.push(name);
+    if (stack2.parent)
+      pushScope(stack2.parent);
+  }
+  pushScope(stack);
+  return scopes;
+}
+function getGrammarStack(state, theme) {
+  if (!(state instanceof GrammarState))
+    throw new ShikiError("Invalid grammar state");
+  return state.getInternalStack(theme);
+}
+
 function transformerDecorations() {
   const map = /* @__PURE__ */ new WeakMap();
   function getContext(shiki) {
@@ -350,51 +445,6 @@ function getTransformers(options) {
   ];
 }
 
-class GrammarState {
-  constructor(_stack, lang, theme) {
-    this._stack = _stack;
-    this.lang = lang;
-    this.theme = theme;
-  }
-  /**
-   * Static method to create a initial grammar state.
-   */
-  static initial(lang, theme) {
-    return new GrammarState(INITIAL, lang, theme);
-  }
-  get scopes() {
-    return getScopes(this._stack);
-  }
-  toJSON() {
-    return {
-      lang: this.lang,
-      theme: this.theme,
-      scopes: this.scopes
-    };
-  }
-}
-function getScopes(stack) {
-  const scopes = [];
-  const visited = /* @__PURE__ */ new Set();
-  function pushScope(stack2) {
-    if (visited.has(stack2))
-      return;
-    visited.add(stack2);
-    const name = stack2?.nameScopesList?.scopeName;
-    if (name)
-      scopes.push(name);
-    if (stack2.parent)
-      pushScope(stack2.parent);
-  }
-  pushScope(stack);
-  return scopes;
-}
-function getGrammarStack(state) {
-  if (!(state instanceof GrammarState))
-    throw new ShikiError("Invalid grammar state");
-  return state._stack;
-}
-
 // src/colors.ts
 var namedColors = [
   "black",
@@ -735,13 +785,17 @@ function codeToTokensBase(internal, code, options = {}) {
     if (options.grammarState.lang !== _grammar.name) {
       throw new ShikiError$1(`Grammar state language "${options.grammarState.lang}" does not match highlight language "${_grammar.name}"`);
     }
-    if (options.grammarState.theme !== themeName) {
-      throw new ShikiError$1(`Grammar state theme "${options.grammarState.theme}" does not match highlight theme "${themeName}"`);
+    if (!options.grammarState.themes.includes(theme.name)) {
+      throw new ShikiError$1(`Grammar state themes "${options.grammarState.themes}" do not contain highlight theme "${theme.name}"`);
     }
   }
   return tokenizeWithTheme(code, _grammar, theme, colorMap, options);
 }
-function getLastGrammarState(internal, code, options = {}) {
+function getLastGrammarState(...args) {
+  if (args.length === 2) {
+    return getLastGrammarStateFromMap(args[1]);
+  }
+  const [internal, code, options = {}] = args;
   const {
     lang = "text",
     theme: themeName = internal.getLoadedThemes()[0]
@@ -759,7 +813,14 @@ function getLastGrammarState(internal, code, options = {}) {
   );
 }
 function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
-  return _tokenizeWithTheme(code, grammar, theme, colorMap, options).tokens;
+  const result = _tokenizeWithTheme(code, grammar, theme, colorMap, options);
+  const grammarState = new GrammarState(
+    _tokenizeWithTheme(code, grammar, theme, colorMap, options).stateStack,
+    grammar.name,
+    theme.name
+  );
+  setLastGrammarStateToMap(result.tokens, grammarState);
+  return result.tokens;
 }
 function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
   const colorReplacements = resolveColorReplacements(theme, options);
@@ -768,7 +829,7 @@ function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     tokenizeTimeLimit = 500
   } = options;
   const lines = splitLines(code);
-  let stateStack = options.grammarState ? getGrammarStack(options.grammarState) : options.grammarContextCode != null ? _tokenizeWithTheme(
+  let stateStack = options.grammarState ? getGrammarStack(options.grammarState, theme.name) ?? INITIAL : options.grammarContextCode != null ? _tokenizeWithTheme(
     options.grammarContextCode,
     grammar,
     theme,
@@ -924,11 +985,21 @@ function explainThemeScope(themeSettingsSelectors, scope, parentScopes) {
 
 function codeToTokensWithThemes(internal, code, options) {
   const themes = Object.entries(options.themes).filter((i) => i[1]).map((i) => ({ color: i[0], theme: i[1] }));
-  const tokens = syncThemesTokenization(
-    ...themes.map((t) => codeToTokensBase(internal, code, {
+  const themedTokens = themes.map((t) => {
+    const tokens2 = codeToTokensBase(internal, code, {
       ...options,
       theme: t.theme
-    }))
+    });
+    const state = getLastGrammarStateFromMap(tokens2);
+    const theme = typeof t.theme === "string" ? t.theme : t.theme.name;
+    return {
+      tokens: tokens2,
+      state,
+      theme
+    };
+  });
+  const tokens = syncThemesTokenization(
+    ...themedTokens.map((i) => i.tokens)
   );
   const mergedTokens = tokens[0].map(
     (line, lineIdx) => line.map((_token, tokenIdx) => {
@@ -952,6 +1023,12 @@ function codeToTokensWithThemes(internal, code, options) {
       return mergedToken;
     })
   );
+  const mergedGrammarState = themedTokens[0].state ? new GrammarState(
+    Object.fromEntries(themedTokens.map((s) => [s.theme, s.state?.getInternalStack(s.theme)])),
+    themedTokens[0].state.lang
+  ) : void 0;
+  if (mergedGrammarState)
+    setLastGrammarStateToMap(mergedTokens, mergedGrammarState);
   return mergedTokens;
 }
 function syncThemesTokenization(...themes) {
@@ -994,6 +1071,7 @@ function codeToTokens(internal, code, options) {
   let tokens;
   let themeName;
   let rootStyle;
+  let grammarState;
   if ("themes" in options) {
     const {
       defaultColor = "light",
@@ -1001,17 +1079,20 @@ function codeToTokens(internal, code, options) {
     } = options;
     const themes = Object.entries(options.themes).filter((i) => i[1]).map((i) => ({ color: i[0], theme: i[1] })).sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0);
     if (themes.length === 0)
-      throw new ShikiError("`themes` option must not be empty");
+      throw new ShikiError$1("`themes` option must not be empty");
     const themeTokens = codeToTokensWithThemes(
       internal,
       code,
       options
     );
+    grammarState = getLastGrammarStateFromMap(themeTokens);
     if (defaultColor && !themes.find((t) => t.color === defaultColor))
-      throw new ShikiError(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``);
+      throw new ShikiError$1(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``);
     const themeRegs = themes.map((t) => internal.getTheme(t.theme));
     const themesOrder = themes.map((t) => t.color);
     tokens = themeTokens.map((line) => line.map((token) => mergeToken(token, themesOrder, cssVariablePrefix, defaultColor)));
+    if (grammarState)
+      setLastGrammarStateToMap(tokens, grammarState);
     const themeColorReplacements = themes.map((t) => resolveColorReplacements(t.theme, options));
     fg = themes.map((t, idx) => (idx === 0 && defaultColor ? "" : `${cssVariablePrefix + t.color}:`) + (applyColorReplacements(themeRegs[idx].fg, themeColorReplacements[idx]) || "inherit")).join(";");
     bg = themes.map((t, idx) => (idx === 0 && defaultColor ? "" : `${cssVariablePrefix + t.color}-bg:`) + (applyColorReplacements(themeRegs[idx].bg, themeColorReplacements[idx]) || "inherit")).join(";");
@@ -1028,15 +1109,17 @@ function codeToTokens(internal, code, options) {
     bg = applyColorReplacements(_theme.bg, colorReplacements);
     fg = applyColorReplacements(_theme.fg, colorReplacements);
     themeName = _theme.name;
+    grammarState = getLastGrammarStateFromMap(tokens);
   } else {
-    throw new ShikiError("Invalid options, either `theme` or `themes` must be provided");
+    throw new ShikiError$1("Invalid options, either `theme` or `themes` must be provided");
   }
   return {
     tokens,
     fg,
     bg,
     themeName,
-    rootStyle
+    rootStyle,
+    grammarState
   };
 }
 function mergeToken(merged, variantsOrder, cssVariablePrefix, defaultColor) {
@@ -1078,7 +1161,8 @@ function codeToHast(internal, code, options, transformerContext = {
     fg,
     bg,
     themeName,
-    rootStyle
+    rootStyle,
+    grammarState
   } = codeToTokens(internal, input, options);
   const {
     mergeWhitespaces = true
@@ -1104,10 +1188,11 @@ function codeToHast(internal, code, options, transformerContext = {
       themeName,
       rootStyle
     },
-    contextSource
+    contextSource,
+    grammarState
   );
 }
-function tokensToHast(tokens, options, transformerContext) {
+function tokensToHast(tokens, options, transformerContext, grammarState = getLastGrammarStateFromMap(tokens)) {
   const transformers = getTransformers(options);
   const lines = [];
   const root = {
@@ -1222,6 +1307,8 @@ function tokensToHast(tokens, options, transformerContext) {
   let result = root;
   for (const transformer of transformers)
     result = transformer?.root?.call(context, result) || result;
+  if (grammarState)
+    setLastGrammarStateToMap(result, grammarState);
   return result;
 }
 function mergeWhitespaceTokens(tokens) {
@@ -1731,7 +1818,7 @@ function getShikiInternal(options = {}) {
 async function createHighlighterCore(options = {}) {
   const internal = await createShikiInternal(options);
   return {
-    getLastGrammarState: (code, options2) => getLastGrammarState(internal, code, options2),
+    getLastGrammarState: (...args) => getLastGrammarState(internal, ...args),
     codeToTokensBase: (code, options2) => codeToTokensBase(internal, code, options2),
     codeToTokensWithThemes: (code, options2) => codeToTokensWithThemes(internal, code, options2),
     codeToTokens: (code, options2) => codeToTokens(internal, code, options2),
@@ -1744,7 +1831,7 @@ async function createHighlighterCore(options = {}) {
 function createHighlighterCoreSync(options = {}) {
   const internal = createShikiInternalSync(options);
   return {
-    getLastGrammarState: (code, options2) => getLastGrammarState(internal, code, options2),
+    getLastGrammarState: (...args) => getLastGrammarState(internal, ...args),
     codeToTokensBase: (code, options2) => codeToTokensBase(internal, code, options2),
     codeToTokensWithThemes: (code, options2) => codeToTokensWithThemes(internal, code, options2),
     codeToTokens: (code, options2) => codeToTokens(internal, code, options2),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.21.0",
+  "version": "1.22.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",
@@ -60,12 +60,12 @@
     "dist"
   ],
   "dependencies": {
-    "@shikijs/vscode-textmate": "^9.2.2",
+    "@shikijs/vscode-textmate": "^9.3.0",
     "@types/hast": "^3.0.4",
     "hast-util-to-html": "^9.0.3",
-    "@shikijs/engine-javascript": "1.21.0",
-    "@shikijs/engine-oniguruma": "1.21.0",
-    "@shikijs/types": "1.21.0"
+    "@shikijs/engine-oniguruma": "1.22.0",
+    "@shikijs/types": "1.22.0",
+    "@shikijs/engine-javascript": "1.22.0"
   },
   "scripts": {
     "build": "unbuild",