@shikijs/core 1.9.0 → 1.10.0

package/dist/chunk-tokens.d.mts

@@ -852,6 +852,24 @@ interface BundledThemeInfo {
     import: DynamicImportThemeRegistration;
 }
 
+/**
+ * GrammarState is a special reference object that holds the state of a grammar.
+ *
+ * It's used to highlight code snippets that are part of the target language.
+ */
+declare class GrammarState {
+    private _stack;
+    lang: string;
+    theme: string;
+    constructor(_stack: StateStack, lang: string, theme: string);
+    get scopes(): string[];
+    toJSON(): {
+        lang: string;
+        theme: string;
+        scopes: string[];
+    };
+}
+
 interface TransformerOptions {
     /**
      * Transformers for the Shiki pipeline.
@@ -1087,7 +1105,7 @@ interface CodeOptionsMultipleThemes<Themes extends string = string> {
 }
 type CodeOptionsThemes<Themes extends string = string> = CodeOptionsSingleTheme<Themes> | CodeOptionsMultipleThemes<Themes>;
 type CodeToHastOptions<Languages extends string = string, Themes extends string = string> = CodeToHastOptionsCommon<Languages> & CodeOptionsThemes<Themes> & CodeOptionsMeta;
-interface CodeToHastOptionsCommon<Languages extends string = string> extends TransformerOptions, DecorationOptions, Pick<TokenizeWithThemeOptions, 'colorReplacements' | 'tokenizeMaxLineLength' | 'tokenizeTimeLimit'> {
+interface CodeToHastOptionsCommon<Languages extends string = string> extends TransformerOptions, DecorationOptions, Pick<TokenizeWithThemeOptions, 'colorReplacements' | 'tokenizeMaxLineLength' | 'tokenizeTimeLimit' | 'grammarState' | 'grammarContextCode'> {
     lang: StringLiteralUnion<Languages | SpecialLanguage>;
     /**
      * Merge whitespace tokens to saving extra `<span>`.
@@ -1290,6 +1308,19 @@ interface TokenizeWithThemeOptions {
      * @default 500 (0.5s)
      */
     tokenizeTimeLimit?: number;
+    /**
+     * Represent the state of the grammar, allowing to continue tokenizing from a intermediate grammar state.
+     *
+     * You can get the grammar state from `getLastGrammarState`.
+     */
+    grammarState?: GrammarState;
+    /**
+     * The code context of the grammar.
+     * Consider it a prepended code to the input code, that only participate the grammar inference but not presented in the final output.
+     *
+     * This will be ignored if `grammarState` is provided.
+     */
+    grammarContextCode?: string;
 }
 /**
  * Result of `codeToTokens`, an object with 2D array of tokens and meta info like background and foreground color.
@@ -1325,4 +1356,4 @@ declare enum FontStyle {
     Underline = 4
 }
 
-export { type ThemeRegistration as $, type RootContent as A, type BundledHighlighterOptions as B, type CodeToHastOptions as C, type ShikiTransformer as D, type Element as E, FontStyle as F, type AnsiLanguage as G, type HighlighterCoreOptions as H, INITIAL as I, type ResolveBundleKey as J, type LanguageRegistration as K, type LanguageInput as L, type MaybeArray as M, type Nodes as N, type BundledLanguageInfo as O, type PlainTextLanguage as P, type DynamicImportLanguageRegistration as Q, Registry as R, type StateStack as S, Theme as T, type CodeOptionsSingleTheme as U, type CodeOptionsMultipleThemes as V, type CodeOptionsThemes as W, type CodeToHastOptionsCommon as X, type CodeOptionsMeta as Y, type CodeToHastRenderOptionsCommon as Z, type ThemeRegistrationRaw as _, type IRawTheme as a, type DynamicImportThemeRegistration as a0, type BundledThemeInfo as a1, type ThemedTokenScopeExplanation as a2, type ThemedTokenExplanation as a3, type TokenBase as a4, type TransformerOptions as a5, type ShikiTransformerContextMeta as a6, type ShikiTransformerContext as a7, type Awaitable as a8, type MaybeGetter as a9, type MaybeModule as aa, type StringLiteralUnion as ab, type DecorationOptions as ac, type DecorationItem as ad, type ResolvedDecorationItem as ae, type DecorationTransformType as af, type Offset as ag, type OffsetOrPosition as ah, type ResolvedPosition as ai, type IRawGrammar as b, type IGrammar as c, type IGrammarConfiguration as d, type IOnigLib as e, type RegistryOptions as f, type IRawThemeSetting as g, type ThemeInput as h, type Root as i, type CodeToTokensOptions as j, type TokensResult as k, type RequireKeys as l, type CodeToTokensBaseOptions as m, type ThemedToken as n, type CodeToTokensWithThemesOptions as o, type ThemedTokenWithVariants as p, type SpecialLanguage as q, type SpecialTheme as r, type ThemeRegistrationAny as s, type TokenizeWithThemeOptions as t, type TokenStyles as u, type Position as v, type ThemeRegistrationResolved as w, type ShikiTransformerContextCommon as x, type CodeToHastRenderOptions as y, type ShikiTransformerContextSource as z };
+export { type ThemeRegistrationRaw as $, type RootContent as A, type BundledHighlighterOptions as B, type CodeToHastOptions as C, type ShikiTransformer as D, type Element as E, FontStyle as F, GrammarState as G, type HighlighterCoreOptions as H, INITIAL as I, type AnsiLanguage as J, type ResolveBundleKey as K, type LanguageInput as L, type MaybeArray as M, type Nodes as N, type LanguageRegistration as O, type PlainTextLanguage as P, type BundledLanguageInfo as Q, Registry as R, type StateStack as S, Theme as T, type DynamicImportLanguageRegistration as U, type CodeOptionsSingleTheme as V, type CodeOptionsMultipleThemes as W, type CodeOptionsThemes as X, type CodeToHastOptionsCommon as Y, type CodeOptionsMeta as Z, type CodeToHastRenderOptionsCommon as _, type IRawTheme as a, type ThemeRegistration as a0, type DynamicImportThemeRegistration as a1, type BundledThemeInfo as a2, type ThemedTokenScopeExplanation as a3, type ThemedTokenExplanation as a4, type TokenBase as a5, type TransformerOptions as a6, type ShikiTransformerContextMeta as a7, type ShikiTransformerContext as a8, type Awaitable as a9, type MaybeGetter as aa, type MaybeModule as ab, type StringLiteralUnion as ac, type DecorationOptions as ad, type DecorationItem as ae, type ResolvedDecorationItem as af, type DecorationTransformType as ag, type Offset as ah, type OffsetOrPosition as ai, type ResolvedPosition as aj, type IRawGrammar as b, type IGrammar as c, type IGrammarConfiguration as d, type IOnigLib as e, type RegistryOptions as f, type IRawThemeSetting as g, type ThemeInput as h, type Root as i, type CodeToTokensOptions as j, type TokensResult as k, type RequireKeys as l, type CodeToTokensBaseOptions as m, type ThemedToken as n, type CodeToTokensWithThemesOptions as o, type ThemedTokenWithVariants as p, type SpecialLanguage as q, type SpecialTheme as r, type ThemeRegistrationAny as s, type TokenizeWithThemeOptions as t, type TokenStyles as u, type Position as v, type ThemeRegistrationResolved as w, type ShikiTransformerContextCommon as x, type CodeToHastRenderOptions as y, type ShikiTransformerContextSource as z };

package/dist/index.d.mts

@@ -1,6 +1,7 @@
 import { HighlighterCore, HighlighterGeneric, ShikiInternal } from './types.mjs';
-import { H as HighlighterCoreOptions, B as BundledHighlighterOptions, L as LanguageInput, h as ThemeInput, C as CodeToHastOptions, i as Root, j as CodeToTokensOptions, k as TokensResult, l as RequireKeys, m as CodeToTokensBaseOptions, n as ThemedToken, o as CodeToTokensWithThemesOptions, p as ThemedTokenWithVariants, M as MaybeArray, P as PlainTextLanguage, q as SpecialLanguage, r as SpecialTheme, E as Element, s as ThemeRegistrationAny, t as TokenizeWithThemeOptions, u as TokenStyles, v as Position, c as IGrammar, w as ThemeRegistrationResolved, x as ShikiTransformerContextCommon, y as CodeToHastRenderOptions, z as ShikiTransformerContextSource, N as Nodes$1, A as RootContent$1, D as ShikiTransformer } from './chunk-tokens.mjs';
-export { G as AnsiLanguage, a8 as Awaitable, O as BundledLanguageInfo, a1 as BundledThemeInfo, Y as CodeOptionsMeta, V as CodeOptionsMultipleThemes, U as CodeOptionsSingleTheme, W as CodeOptionsThemes, X as CodeToHastOptionsCommon, Z as CodeToHastRenderOptionsCommon, ad as DecorationItem, ac as DecorationOptions, af as DecorationTransformType, Q as DynamicImportLanguageRegistration, a0 as DynamicImportThemeRegistration, F as FontStyle, K as LanguageRegistration, a9 as MaybeGetter, aa as MaybeModule, ag as Offset, ah as OffsetOrPosition, b as RawGrammar, a as RawTheme, g as RawThemeSetting, J as ResolveBundleKey, ae as ResolvedDecorationItem, ai as ResolvedPosition, a7 as ShikiTransformerContext, a6 as ShikiTransformerContextMeta, ab as StringLiteralUnion, $ as ThemeRegistration, _ as ThemeRegistrationRaw, a3 as ThemedTokenExplanation, a2 as ThemedTokenScopeExplanation, a4 as TokenBase, a5 as TransformerOptions } from './chunk-tokens.mjs';
+export { Grammar } from './types.mjs';
+import { H as HighlighterCoreOptions, B as BundledHighlighterOptions, L as LanguageInput, h as ThemeInput, C as CodeToHastOptions, i as Root, j as CodeToTokensOptions, k as TokensResult, l as RequireKeys, m as CodeToTokensBaseOptions, n as ThemedToken, o as CodeToTokensWithThemesOptions, p as ThemedTokenWithVariants, G as GrammarState, M as MaybeArray, P as PlainTextLanguage, q as SpecialLanguage, r as SpecialTheme, E as Element, s as ThemeRegistrationAny, t as TokenizeWithThemeOptions, u as TokenStyles, v as Position, c as IGrammar, w as ThemeRegistrationResolved, x as ShikiTransformerContextCommon, y as CodeToHastRenderOptions, z as ShikiTransformerContextSource, N as Nodes$1, A as RootContent$1, D as ShikiTransformer } from './chunk-tokens.mjs';
+export { J as AnsiLanguage, a9 as Awaitable, Q as BundledLanguageInfo, a2 as BundledThemeInfo, Z as CodeOptionsMeta, W as CodeOptionsMultipleThemes, V as CodeOptionsSingleTheme, X as CodeOptionsThemes, Y as CodeToHastOptionsCommon, _ as CodeToHastRenderOptionsCommon, ae as DecorationItem, ad as DecorationOptions, ag as DecorationTransformType, U as DynamicImportLanguageRegistration, a1 as DynamicImportThemeRegistration, F as FontStyle, O as LanguageRegistration, aa as MaybeGetter, ab as MaybeModule, ah as Offset, ai as OffsetOrPosition, b as RawGrammar, a as RawTheme, g as RawThemeSetting, K as ResolveBundleKey, af as ResolvedDecorationItem, aj as ResolvedPosition, a8 as ShikiTransformerContext, a7 as ShikiTransformerContextMeta, ac as StringLiteralUnion, a0 as ThemeRegistration, $ as ThemeRegistrationRaw, a4 as ThemedTokenExplanation, a3 as ThemedTokenScopeExplanation, a5 as TokenBase, a6 as TransformerOptions } from './chunk-tokens.mjs';
 import { L as LoadWasmOptions } from './chunk-index.mjs';
 export { W as WebAssemblyInstantiator, l as loadWasm } from './chunk-index.mjs';
 
@@ -64,9 +65,14 @@ interface ShorthandsBundle<L extends string, T extends string> {
      */
     codeToTokensWithThemes: (code: string, options: RequireKeys<CodeToTokensWithThemesOptions<L, T>, 'themes' | 'lang'>) => Promise<ThemedTokenWithVariants[][]>;
     /**
-     * Get internal singleton highlighter.
+     * Get the singleton highlighter.
      */
     getSingletonHighlighter: (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
+    /**
+     * Shorthand for `getLastGrammarState` with auto-loaded theme and language.
+     * A singleton highlighter it maintained internally.
+     */
+    getLastGrammarState: (code: string, options: CodeToTokensBaseOptions<L, T>) => Promise<GrammarState>;
 }
 declare function makeSingletonHighlighter<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): (options?: Partial<BundledHighlighterOptions<L, T>>) => Promise<HighlighterGeneric<L, T>>;
 declare function createSingletonShorthands<L extends string, T extends string>(createHighlighter: CreateHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
@@ -410,4 +416,4 @@ declare class ShikiError extends Error {
     constructor(message: string);
 }
 
-export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, IGrammar as Grammar, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, Position, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createPositionConverter, createShikiInternal, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };
+export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, GrammarState, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, Position, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createPositionConverter, createShikiInternal, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { StackElementMetadata, INITIAL, Registry as Registry$1, Theme } from './textmate.mjs';
+import { INITIAL, StackElementMetadata, Registry as Registry$1, Theme } from './textmate.mjs';
 import { FontStyle } from './types.mjs';
 
 function toArray(x) {
@@ -517,6 +517,61 @@ function dimColor(color) {
     return color;
 }
 
+class ShikiError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ShikiError';
+    }
+}
+
+/**
+ * GrammarState is a special reference object that holds the state of a grammar.
+ *
+ * It's used to highlight code snippets that are part of the target language.
+ */
+class GrammarState {
+    _stack;
+    lang;
+    theme;
+    constructor(_stack, lang, theme) {
+        this._stack = _stack;
+        this.lang = lang;
+        this.theme = theme;
+    }
+    get scopes() {
+        return getScopes(this._stack);
+    }
+    toJSON() {
+        return {
+            lang: this.lang,
+            theme: this.theme,
+            scopes: this.scopes,
+        };
+    }
+}
+function getScopes(stack) {
+    const scopes = [];
+    const visited = new Set();
+    function pushScope(stack) {
+        if (visited.has(stack))
+            return;
+        visited.add(stack);
+        const name = stack?.nameScopesList?.scopeName;
+        if (name)
+            scopes.push(name);
+        if (stack.parent)
+            pushScope(stack.parent);
+    }
+    pushScope(stack);
+    return scopes;
+}
+function getGrammarStack(state) {
+    if (!(state instanceof GrammarState))
+        throw new ShikiError('Invalid grammar state');
+    // @ts-expect-error _stack is private
+    return state._stack;
+}
+
 /**
  * Code to tokens, with a simple theme.
  */
@@ -528,13 +583,42 @@ function codeToTokensBase(internal, code, options = {}) {
     if (lang === 'ansi')
         return tokenizeAnsiWithTheme(theme, code, options);
     const _grammar = internal.getLanguage(lang);
+    if (options.grammarState) {
+        if (options.grammarState.lang !== _grammar.name) {
+            throw new ShikiError(`Grammar state language "${options.grammarState.lang}" does not match highlight language "${_grammar.name}"`);
+        }
+        if (options.grammarState.theme !== themeName) {
+            throw new ShikiError(`Grammar state theme "${options.grammarState.theme}" does not match highlight theme "${themeName}"`);
+        }
+    }
     return tokenizeWithTheme(code, _grammar, theme, colorMap, options);
 }
+function getLastGrammarState(internal, code, options = {}) {
+    const { lang = 'text', theme: themeName = internal.getLoadedThemes()[0], } = options;
+    if (isPlainLang(lang) || isNoneTheme(themeName))
+        throw new ShikiError('Plain language does not have grammar state');
+    if (lang === 'ansi')
+        throw new ShikiError('ANSI language does not have grammar state');
+    const { theme, colorMap } = internal.setTheme(themeName);
+    const _grammar = internal.getLanguage(lang);
+    return new GrammarState(_tokenizeWithTheme(code, _grammar, theme, colorMap, options).stateStack, _grammar.name, theme.name);
+}
 function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
+    return _tokenizeWithTheme(code, grammar, theme, colorMap, options).tokens;
+}
+function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     const colorReplacements = resolveColorReplacements(theme, options);
     const { tokenizeMaxLineLength = 0, tokenizeTimeLimit = 500, } = options;
     const lines = splitLines(code);
-    let ruleStack = INITIAL;
+    let stateStack = options.grammarState
+        ? getGrammarStack(options.grammarState)
+        : options.grammarContextCode != null
+            ? _tokenizeWithTheme(options.grammarContextCode, grammar, theme, colorMap, {
+                ...options,
+                grammarState: undefined,
+                grammarContextCode: undefined,
+            }).stateStack
+            : INITIAL;
     let actual = [];
     const final = [];
     const themeSettingsSelectors = [];
@@ -579,11 +663,11 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
         let tokensWithScopes;
         let tokensWithScopesIndex;
         if (options.includeExplanation) {
-            resultWithScopes = grammar.tokenizeLine(line, ruleStack);
+            resultWithScopes = grammar.tokenizeLine(line, stateStack);
             tokensWithScopes = resultWithScopes.tokens;
             tokensWithScopesIndex = 0;
         }
-        const result = grammar.tokenizeLine2(line, ruleStack, tokenizeTimeLimit);
+        const result = grammar.tokenizeLine2(line, stateStack, tokenizeTimeLimit);
         const tokensLength = result.tokens.length / 2;
         for (let j = 0; j < tokensLength; j++) {
             const startIndex = result.tokens[2 * j];
@@ -617,9 +701,12 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
         }
         final.push(actual);
         actual = [];
-        ruleStack = result.ruleStack;
+        stateStack = result.ruleStack;
     }
-    return final;
+    return {
+        tokens: final,
+        stateStack,
+    };
 }
 function explainThemeScopes(themeSelectors, scopes) {
     const result = [];
@@ -739,13 +826,6 @@ function syncThemesTokenization(...themes) {
     return outThemes;
 }
 
-class ShikiError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = 'ShikiError';
-    }
-}
-
 /**
  * High-level code-to-tokens API.
  *
@@ -5274,6 +5354,7 @@ class Registry extends Registry$1 {
         // @ts-expect-error Private members, set this to override the previous grammar (that can be a stub)
         this._syncRegistry._rawGrammars.set(lang.scopeName, lang);
         const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig);
+        g.name = lang.name;
         this._resolvedGrammars.set(lang.name, g);
         if (lang.aliases) {
             lang.aliases.forEach((alias) => {
@@ -5516,6 +5597,7 @@ function getShikiInternal(options = {}) {
 async function createHighlighterCore(options = {}) {
     const internal = await createShikiInternal(options);
     return {
+        getLastGrammarState: (code, options) => getLastGrammarState(internal, code, options),
         codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options),
         codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options),
         codeToTokens: (code, options) => codeToTokens(internal, code, options),
@@ -5672,6 +5754,13 @@ function createSingletonShorthands(createHighlighter) {
             });
             return shiki.codeToTokensWithThemes(code, options);
         },
+        async getLastGrammarState(code, options) {
+            const shiki = await getSingletonHighlighter({
+                langs: [options.lang],
+                themes: [options.theme],
+            });
+            return shiki.getLastGrammarState(code, options);
+        },
     };
 }
 

package/dist/types.d.mts

@@ -1,7 +1,11 @@
-import { h as ThemeInput, r as SpecialTheme, L as LanguageInput, q as SpecialLanguage, s as ThemeRegistrationAny, w as ThemeRegistrationResolved, K as LanguageRegistration, c as IGrammar, C as CodeToHastOptions, J as ResolveBundleKey, i as Root, j as CodeToTokensOptions, k as TokensResult, m as CodeToTokensBaseOptions, n as ThemedToken, o as CodeToTokensWithThemesOptions, p as ThemedTokenWithVariants } from './chunk-tokens.mjs';
-export { G as AnsiLanguage, a8 as Awaitable, B as BundledHighlighterOptions, O as BundledLanguageInfo, a1 as BundledThemeInfo, Y as CodeOptionsMeta, V as CodeOptionsMultipleThemes, U as CodeOptionsSingleTheme, W as CodeOptionsThemes, X as CodeToHastOptionsCommon, y as CodeToHastRenderOptions, Z as CodeToHastRenderOptionsCommon, ad as DecorationItem, ac as DecorationOptions, af as DecorationTransformType, Q as DynamicImportLanguageRegistration, a0 as DynamicImportThemeRegistration, F as FontStyle, H as HighlighterCoreOptions, M as MaybeArray, a9 as MaybeGetter, aa as MaybeModule, ag as Offset, ah as OffsetOrPosition, P as PlainTextLanguage, v as Position, b as RawGrammar, a as RawTheme, g as RawThemeSetting, l as RequireKeys, ae as ResolvedDecorationItem, ai as ResolvedPosition, D as ShikiTransformer, a7 as ShikiTransformerContext, x as ShikiTransformerContextCommon, a6 as ShikiTransformerContextMeta, z as ShikiTransformerContextSource, ab as StringLiteralUnion, $ as ThemeRegistration, _ as ThemeRegistrationRaw, a3 as ThemedTokenExplanation, a2 as ThemedTokenScopeExplanation, a4 as TokenBase, u as TokenStyles, t as TokenizeWithThemeOptions, a5 as TransformerOptions } from './chunk-tokens.mjs';
+import { c as IGrammar, h as ThemeInput, r as SpecialTheme, L as LanguageInput, q as SpecialLanguage, s as ThemeRegistrationAny, w as ThemeRegistrationResolved, O as LanguageRegistration, C as CodeToHastOptions, K as ResolveBundleKey, i as Root, j as CodeToTokensOptions, k as TokensResult, m as CodeToTokensBaseOptions, n as ThemedToken, o as CodeToTokensWithThemesOptions, p as ThemedTokenWithVariants, G as GrammarState } from './chunk-tokens.mjs';
+export { J as AnsiLanguage, a9 as Awaitable, B as BundledHighlighterOptions, Q as BundledLanguageInfo, a2 as BundledThemeInfo, Z as CodeOptionsMeta, W as CodeOptionsMultipleThemes, V as CodeOptionsSingleTheme, X as CodeOptionsThemes, Y as CodeToHastOptionsCommon, y as CodeToHastRenderOptions, _ as CodeToHastRenderOptionsCommon, ae as DecorationItem, ad as DecorationOptions, ag as DecorationTransformType, U as DynamicImportLanguageRegistration, a1 as DynamicImportThemeRegistration, F as FontStyle, H as HighlighterCoreOptions, M as MaybeArray, aa as MaybeGetter, ab as MaybeModule, ah as Offset, ai as OffsetOrPosition, P as PlainTextLanguage, v as Position, b as RawGrammar, a as RawTheme, g as RawThemeSetting, l as RequireKeys, af as ResolvedDecorationItem, aj as ResolvedPosition, D as ShikiTransformer, a8 as ShikiTransformerContext, x as ShikiTransformerContextCommon, a7 as ShikiTransformerContextMeta, z as ShikiTransformerContextSource, ac as StringLiteralUnion, a0 as ThemeRegistration, $ as ThemeRegistrationRaw, a4 as ThemedTokenExplanation, a3 as ThemedTokenScopeExplanation, a5 as TokenBase, u as TokenStyles, t as TokenizeWithThemeOptions, a6 as TransformerOptions } from './chunk-tokens.mjs';
 export { W as WebAssemblyInstantiator } from './chunk-index.mjs';
 
+interface Grammar extends IGrammar {
+    name: string;
+}
+
 /**
  * Internal context of Shiki, core textmate logic
  */
@@ -21,7 +25,7 @@ interface ShikiInternal<BundledLangKeys extends string = never, BundledThemeKeys
     /**
      * Get the registered language object
      */
-    getLanguage: (name: string | LanguageRegistration) => IGrammar;
+    getLanguage: (name: string | LanguageRegistration) => Grammar;
     /**
      * Set the current theme and get the resolved theme object and color map.
      * @internal
@@ -81,6 +85,11 @@ interface HighlighterGeneric<BundledLangKeys extends string, BundledThemeKeys ex
      * @returns A 2D array of tokens, first dimension is lines, second dimension is tokens in a line.
      */
     codeToTokensWithThemes: (code: string, options: CodeToTokensWithThemesOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>) => ThemedTokenWithVariants[][];
+    /**
+     * Get the last grammar state of a code snippet.
+     * You can pass the grammar state to `codeToTokens` as `grammarState` to continue tokenizing from an intermediate state.
+     */
+    getLastGrammarState: (code: string, options: CodeToTokensBaseOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>) => GrammarState;
     /**
      * Get internal context object
      * @internal
@@ -93,4 +102,4 @@ interface HighlighterGeneric<BundledLangKeys extends string, BundledThemeKeys ex
  */
 type HighlighterCore = HighlighterGeneric<never, never>;
 
-export { CodeToHastOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, IGrammar as Grammar, type HighlighterCore, type HighlighterGeneric, LanguageInput, LanguageRegistration, ResolveBundleKey, type ShikiInternal, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokensResult };
+export { CodeToHastOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type Grammar, GrammarState, type HighlighterCore, type HighlighterGeneric, LanguageInput, LanguageRegistration, ResolveBundleKey, type ShikiInternal, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokensResult };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",