@shikijs/core 1.0.0-beta.1 → 1.0.0-beta.2

package/dist/chunk-types.d.mts

@@ -681,18 +681,26 @@ interface HighlighterGeneric<BundledLangKeys extends string, BundledThemeKeys ex
      */
     codeToHast(code: string, options: CodeToHastOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): Root;
     /**
-     * Get highlighted code in tokens.
+     * Get highlighted code in tokens. Uses `codeToTokensWithThemes` or `codeToTokensBase` based on the options.
+     */
+    codeToTokens(code: string, options: CodeToTokensOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): TokensResult;
+    /**
+     * Get highlighted code in tokens with a single theme.
      * @returns A 2D array of tokens, first dimension is lines, second dimension is tokens in a line.
      */
-    codeToThemedTokens(code: string, options: CodeToThemedTokensOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): ThemedToken[][];
+    codeToTokensBase(code: string, options: CodeToTokensBaseOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): ThemedToken[][];
     /**
      * Get highlighted code in tokens with multiple themes.
      *
-     * Different from `codeToThemedTokens`, each token will have a `variants` property consisting of an object of color name to token styles.
+     * Different from `codeToTokensBase`, each token will have a `variants` property consisting of an object of color name to token styles.
      *
      * @returns A 2D array of tokens, first dimension is lines, second dimension is tokens in a line.
      */
     codeToTokensWithThemes(code: string, options: CodeToTokensWithThemesOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): ThemedTokenWithVariants[][];
+    /**
+     * @deprecated Renamed to `codeToTokensBase`, or use high-level `codeToTokens` directly.
+     */
+    codeToThemedTokens(code: string, options: CodeToTokensBaseOptions<ResolveBundleKey<BundledLangKeys>, ResolveBundleKey<BundledThemeKeys>>): ThemedToken[][];
     /**
      * Load a theme to the highlighter, so later it can be used synchronously.
      */
@@ -799,10 +807,11 @@ interface LanguageRegistration extends IRawGrammar {
      */
     injectTo?: string[];
 }
-interface CodeToThemedTokensOptions<Languages = string, Themes = string> extends TokenizeWithThemeOptions {
+interface CodeToTokensBaseOptions<Languages extends string = string, Themes extends string = string> extends TokenizeWithThemeOptions {
     lang?: Languages | SpecialLanguage;
     theme?: Themes | ThemeRegistrationAny | SpecialTheme;
 }
+type CodeToTokensOptions<Languages extends string = string, Themes extends string = string> = Omit<CodeToTokensBaseOptions<Languages, Themes>, 'theme'> & CodeOptionsThemes<Themes>;
 interface CodeToHastOptionsCommon<Languages extends string = string> extends TransformerOptions, Pick<TokenizeWithThemeOptions, 'colorReplacements'> {
     lang: StringLiteralUnion<Languages | SpecialLanguage>;
     /**
@@ -1000,6 +1009,7 @@ interface ShikiTransformerContextCommon {
     meta: ShikiTransformerContextMeta;
     options: CodeToHastOptions;
     codeToHast: (code: string, options: CodeToHastOptions) => Root;
+    codeToTokens: (code: string, options: CodeToTokensOptions) => TokensResult;
 }
 /**
  * Transformer context for HAST related hooks
@@ -1010,6 +1020,12 @@ interface ShikiTransformerContext extends ShikiTransformerContextCommon {
     readonly pre: Element;
     readonly code: Element;
     readonly lines: Element[];
+    /**
+     * Utility to append class to a hast node
+     *
+     * If the `property.class` is a string, it will be splitted by space and converted to an array.
+     */
+    addClassToHast: (hast: Element, className: string | string[]) => Element;
 }
 interface ShikiTransformer {
     /**
@@ -1058,11 +1074,22 @@ interface ShikiTransformer {
      */
     token?(this: ShikiTransformerContext, hast: Element, line: number, col: number, lineElement: Element): Element | void;
 }
-interface HtmlRendererOptionsCommon extends TransformerOptions {
-    lang?: string;
-    langId?: string;
+interface TokensResult {
+    /**
+     * 2D array of tokens, first dimension is lines, second dimension is tokens in a line.
+     */
+    tokens: ThemedToken[][];
+    /**
+     * Foreground color of the code.
+     */
     fg?: string;
+    /**
+     * Background color of the code.
+     */
     bg?: string;
+    /**
+     * A string representation of themes applied to the token.
+     */
     themeName?: string;
     /**
      * Custom style string to be applied to the root `<pre>` element.
@@ -1070,7 +1097,11 @@ interface HtmlRendererOptionsCommon extends TransformerOptions {
      */
     rootStyle?: string;
 }
-type HtmlRendererOptions = HtmlRendererOptionsCommon & CodeToHastOptions;
+interface CodeToHastRenderOptionsCommon extends TransformerOptions, Omit<TokensResult, 'tokens'> {
+    lang?: string;
+    langId?: string;
+}
+type CodeToHastRenderOptions = CodeToHastRenderOptionsCommon & CodeToHastOptions;
 interface ThemedTokenScopeExplanation {
     scopeName: string;
     themeMatches: any[];
@@ -1227,4 +1258,4 @@ interface ShikijiTransformerContext extends ShikiTransformerContext {
 interface ShikijiTransformerContextCommon extends ShikiTransformerContextCommon {
 }
 
-export { type BundledLanguageInfo as $, type AnsiLanguage as A, type BundledHighlighterOptions as B, type CodeToHastOptions as C, type TransformerOptions as D, type Element as E, FontStyle as F, type ThemeRegistrationRaw as G, type HighlighterGeneric as H, type IGrammar as I, type ThemeRegistration as J, type ShikiTransformerContextMeta as K, type LanguageInput as L, type MaybeArray as M, type Nodes as N, type ShikiTransformerContext as O, type PlainTextLanguage as P, type ShikiTransformer as Q, type Root as R, type SpecialTheme as S, type ThemeInput as T, type HtmlRendererOptionsCommon as U, type ThemedTokenScopeExplanation as V, type ThemedTokenExplanation as W, type TokenBase as X, type TokenStyles as Y, type DynamicImportLanguageRegistration as Z, type DynamicImportThemeRegistration as _, type HighlighterCoreOptions as a, type BundledThemeInfo as a0, type ShikijiTransformer as a1, type ShikijiTransformerContext as a2, type ShikijiTransformerContextCommon as a3, Registry as a4, INITIAL as a5, type StateStack as a6, type IRawTheme as a7, type IGrammarConfiguration as a8, type IOnigLib as a9, type RegistryOptions as aa, type IRawThemeSetting as ab, type RequireKeys as b, type CodeToThemedTokensOptions as c, type ThemedToken as d, type CodeToTokensWithThemesOptions as e, type ThemedTokenWithVariants as f, type ShikiInternal as g, type ThemeRegistrationResolved as h, type TokenizeWithThemeOptions as i, type ShikiTransformerContextCommon as j, type HtmlRendererOptions as k, type RootContent as l, type ThemeRegistrationAny as m, type IRawGrammar as n, type SpecialLanguage as o, type Awaitable as p, type MaybeGetter as q, type MaybeModule as r, type StringLiteralUnion as s, type ResolveBundleKey as t, type LanguageRegistration as u, type CodeToHastOptionsCommon as v, type CodeOptionsSingleTheme as w, type CodeOptionsMultipleThemes as x, type CodeOptionsThemes as y, type CodeOptionsMeta as z };
+export { type DynamicImportLanguageRegistration as $, type AnsiLanguage as A, type BundledHighlighterOptions as B, type CodeToHastOptions as C, type CodeOptionsMultipleThemes as D, type Element as E, FontStyle as F, type CodeOptionsThemes as G, type HighlighterGeneric as H, type IGrammar as I, type CodeOptionsMeta as J, type TransformerOptions as K, type LanguageInput as L, type MaybeArray as M, type Nodes as N, type ThemeRegistrationRaw as O, type PlainTextLanguage as P, type ThemeRegistration as Q, type Root as R, type SpecialTheme as S, type ThemeInput as T, type ShikiTransformerContextMeta as U, type ShikiTransformerContext as V, type ShikiTransformer as W, type CodeToHastRenderOptionsCommon as X, type ThemedTokenScopeExplanation as Y, type ThemedTokenExplanation as Z, type TokenBase as _, type HighlighterCoreOptions as a, type DynamicImportThemeRegistration as a0, type BundledLanguageInfo as a1, type BundledThemeInfo as a2, type ShikijiTransformer as a3, type ShikijiTransformerContext as a4, type ShikijiTransformerContextCommon as a5, Registry as a6, INITIAL as a7, type StateStack as a8, type IRawTheme as a9, type IGrammarConfiguration as aa, type IOnigLib as ab, type RegistryOptions as ac, type IRawThemeSetting as ad, type CodeToTokensOptions as b, type TokensResult as c, type RequireKeys as d, type CodeToTokensBaseOptions as e, type ThemedToken as f, type CodeToTokensWithThemesOptions as g, type ThemedTokenWithVariants as h, type TokenStyles as i, type ShikiInternal as j, type ThemeRegistrationResolved as k, type TokenizeWithThemeOptions as l, type ShikiTransformerContextCommon as m, type CodeToHastRenderOptions as n, type RootContent as o, type ThemeRegistrationAny as p, type IRawGrammar as q, type SpecialLanguage as r, type Awaitable as s, type MaybeGetter as t, type MaybeModule as u, type StringLiteralUnion as v, type ResolveBundleKey as w, type LanguageRegistration as x, type CodeToHastOptionsCommon as y, type CodeOptionsSingleTheme as z };

package/dist/index.d.mts

@@ -1,5 +1,5 @@
-import { H as HighlighterGeneric, a as HighlighterCoreOptions, B as BundledHighlighterOptions, L as LanguageInput, T as ThemeInput, C as CodeToHastOptions, R as Root, b as RequireKeys, c as CodeToThemedTokensOptions, d as ThemedToken, e as CodeToTokensWithThemesOptions, f as ThemedTokenWithVariants, M as MaybeArray, S as SpecialTheme, E as Element, g as ShikiInternal, I as IGrammar, h as ThemeRegistrationResolved, i as TokenizeWithThemeOptions, j as ShikiTransformerContextCommon, k as HtmlRendererOptions, N as Nodes$1, l as RootContent$1, m as ThemeRegistrationAny } from './chunk-types.mjs';
-export { A as AnsiLanguage, p as Awaitable, $ as BundledLanguageInfo, a0 as BundledThemeInfo, z as CodeOptionsMeta, x as CodeOptionsMultipleThemes, w as CodeOptionsSingleTheme, y as CodeOptionsThemes, v as CodeToHastOptionsCommon, Z as DynamicImportLanguageRegistration, _ as DynamicImportThemeRegistration, F as FontStyle, U as HtmlRendererOptionsCommon, n as IRawGrammar, u as LanguageRegistration, q as MaybeGetter, r as MaybeModule, P as PlainTextLanguage, n as RawGrammar, t as ResolveBundleKey, Q as ShikiTransformer, O as ShikiTransformerContext, K as ShikiTransformerContextMeta, a1 as ShikijiTransformer, a2 as ShikijiTransformerContext, a3 as ShikijiTransformerContextCommon, o as SpecialLanguage, s as StringLiteralUnion, J as ThemeRegistration, G as ThemeRegistrationRaw, W as ThemedTokenExplanation, V as ThemedTokenScopeExplanation, X as TokenBase, Y as TokenStyles, D as TransformerOptions } from './chunk-types.mjs';
+import { H as HighlighterGeneric, a as HighlighterCoreOptions, B as BundledHighlighterOptions, L as LanguageInput, T as ThemeInput, C as CodeToHastOptions, R as Root, b as CodeToTokensOptions, c as TokensResult, d as RequireKeys, e as CodeToTokensBaseOptions, f as ThemedToken, g as CodeToTokensWithThemesOptions, h as ThemedTokenWithVariants, M as MaybeArray, S as SpecialTheme, E as Element, i as TokenStyles, j as ShikiInternal, I as IGrammar, k as ThemeRegistrationResolved, l as TokenizeWithThemeOptions, m as ShikiTransformerContextCommon, n as CodeToHastRenderOptions, N as Nodes$1, o as RootContent$1, p as ThemeRegistrationAny } from './chunk-types.mjs';
+export { A as AnsiLanguage, s as Awaitable, a1 as BundledLanguageInfo, a2 as BundledThemeInfo, J as CodeOptionsMeta, D as CodeOptionsMultipleThemes, z as CodeOptionsSingleTheme, G as CodeOptionsThemes, y as CodeToHastOptionsCommon, X as CodeToHastRenderOptionsCommon, $ as DynamicImportLanguageRegistration, a0 as DynamicImportThemeRegistration, F as FontStyle, q as IRawGrammar, x as LanguageRegistration, t as MaybeGetter, u as MaybeModule, P as PlainTextLanguage, q as RawGrammar, w as ResolveBundleKey, W as ShikiTransformer, V as ShikiTransformerContext, U as ShikiTransformerContextMeta, a3 as ShikijiTransformer, a4 as ShikijiTransformerContext, a5 as ShikijiTransformerContextCommon, r as SpecialLanguage, v as StringLiteralUnion, Q as ThemeRegistration, O as ThemeRegistrationRaw, Z as ThemedTokenExplanation, Y as ThemedTokenScopeExplanation, _ as TokenBase, K as TransformerOptions } from './chunk-types.mjs';
 import { L as LoadWasmOptions } from './chunk-index.mjs';
 export { l as loadWasm } from './chunk-index.mjs';
 
@@ -37,12 +37,19 @@ interface ShorthandsBundle<L extends string, T extends string> {
      */
     codeToHast(code: string, options: CodeToHastOptions<L, T>): Promise<Root>;
     /**
-     * Shorthand for `codeToThemedTokens` with auto-loaded theme and language.
+     * Shorthand for `codeToTokens` with auto-loaded theme and language.
      * A singleton highlighter it maintained internally.
      *
-     * Differences from `highlighter.codeToThemedTokens()`, this function is async.
+     * Differences from `highlighter.codeToTokens()`, this function is async.
      */
-    codeToThemedTokens(code: string, options: RequireKeys<CodeToThemedTokensOptions<L, T>, 'theme' | 'lang'>): Promise<ThemedToken[][]>;
+    codeToTokens(code: string, options: CodeToTokensOptions<L, T>): Promise<TokensResult>;
+    /**
+     * Shorthand for `codeToTokensBase` with auto-loaded theme and language.
+     * A singleton highlighter it maintained internally.
+     *
+     * Differences from `highlighter.codeToTokensBase()`, this function is async.
+     */
+    codeToTokensBase(code: string, options: RequireKeys<CodeToTokensBaseOptions<L, T>, 'theme' | 'lang'>): Promise<ThemedToken[][]>;
     /**
      * Shorthand for `codeToTokensWithThemes` with auto-loaded theme and language.
      * A singleton highlighter it maintained internally.
@@ -56,6 +63,10 @@ interface ShorthandsBundle<L extends string, T extends string> {
      * @internal
      */
     getSingletonHighlighter(): Promise<HighlighterGeneric<L, T>>;
+    /**
+     * @deprecated Use `codeToTokensBase` instead.
+     */
+    codeToThemedTokens(code: string, options: RequireKeys<CodeToTokensBaseOptions<L, T>, 'theme' | 'lang'>): Promise<ThemedToken[][]>;
 }
 declare function createSingletonShorthands<L extends string, T extends string>(getHighlighter: GetHighlighterFactory<L, T>): ShorthandsBundle<L, T>;
 
@@ -93,7 +104,7 @@ declare function isSpecialTheme(theme: string | ThemeInput | null | undefined):
  *
  * If the `property.class` is a string, it will be splitted by space and converted to an array.
  */
-declare function addClassToHast(node: Element, className: string | string[]): void;
+declare function addClassToHast(node: Element, className: string | string[]): Element;
 /**
  * Split a token into multiple tokens by given offsets.
  *
@@ -105,6 +116,8 @@ declare function applyColorReplacements(color: string, replacements?: Record<str
  * @deprecated Use `isPlainLang` instead.
  */
 declare const isPlaintext: typeof isPlainLang;
+declare function getTokenStyleObject(token: TokenStyles): Record<string, string>;
+declare function stringifyTokenStyle(token: Record<string, string>): string;
 
 /**
  * Set the default wasm loader for `loadWasm`.
@@ -116,13 +129,23 @@ declare function setDefaultWasmLoader(_loader: LoadWasmOptions): void;
  */
 declare function getShikiInternal(options?: HighlighterCoreOptions): Promise<ShikiInternal>;
 
-declare function codeToThemedTokens(internal: ShikiInternal, code: string, options?: CodeToThemedTokensOptions): ThemedToken[][];
+/**
+ * Code to tokens, with a simple theme.
+ */
+declare function codeToTokensBase(internal: ShikiInternal, code: string, options?: CodeToTokensBaseOptions): ThemedToken[][];
 declare function tokenizeWithTheme(code: string, grammar: IGrammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions): ThemedToken[][];
 
+/**
+ * High-level code-to-tokens API.
+ *
+ * It will use `codeToTokensWithThemes` or `codeToTokensBase` based on the options.
+ */
+declare function codeToTokens(internal: ShikiInternal, code: string, options: CodeToTokensOptions): TokensResult;
+
 declare function tokenizeAnsiWithTheme(theme: ThemeRegistrationResolved, fileContents: string, options?: TokenizeWithThemeOptions): ThemedToken[][];
 
 declare function codeToHast(internal: ShikiInternal, code: string, options: CodeToHastOptions, transformerContext?: ShikiTransformerContextCommon): Root;
-declare function tokensToHast(tokens: ThemedToken[][], options: HtmlRendererOptions, transformerContext: ShikiTransformerContextCommon): Root;
+declare function tokensToHast(tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextCommon): Root;
 
 type FormatSmartOptions = {
   /**
@@ -361,4 +384,4 @@ declare function codeToTokensWithThemes(internal: ShikiInternal, code: string, o
  */
 declare function normalizeTheme(rawTheme: ThemeRegistrationAny): ThemeRegistrationResolved;
 
-export { BundledHighlighterOptions, CodeToHastOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, type GetHighlighterFactory, IGrammar as Grammar, type HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, HtmlRendererOptions, IGrammar, LanguageInput, MaybeArray, RequireKeys, ShikiInternal, ShikiTransformerContextCommon, type ShorthandsBundle, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenizeWithThemeOptions, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToThemedTokens, codeToTokensWithThemes, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, toHtml as hastToHtml, isNoneTheme, isPlainLang, isPlaintext, isSpecialLang, isSpecialTheme, normalizeTheme, setDefaultWasmLoader, splitLines, splitToken, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast };
+export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type GetHighlighterFactory, IGrammar as Grammar, type HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, IGrammar, LanguageInput, MaybeArray, RequireKeys, ShikiInternal, ShikiTransformerContextCommon, type ShorthandsBundle, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isPlaintext, isSpecialLang, isSpecialTheme, normalizeTheme, setDefaultWasmLoader, splitLines, splitToken, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast };

package/dist/index.mjs

@@ -49,7 +49,7 @@ function isSpecialTheme(theme) {
  */
 function addClassToHast(node, className) {
     if (!className)
-        return;
+        return node;
     node.properties ||= {};
     node.properties.class ||= [];
     if (typeof node.properties.class === 'string')
@@ -61,6 +61,7 @@ function addClassToHast(node, className) {
         if (c && !node.properties.class.includes(c))
             node.properties.class.push(c);
     }
+    return node;
 }
 /**
  * Split a token into multiple tokens by given offsets.
@@ -96,6 +97,25 @@ function applyColorReplacements(color, replacements) {
  * @deprecated Use `isPlainLang` instead.
  */
 const isPlaintext = isPlainLang;
+function getTokenStyleObject(token) {
+    const styles = {};
+    if (token.color)
+        styles.color = token.color;
+    if (token.bgColor)
+        styles['background-color'] = token.bgColor;
+    if (token.fontStyle) {
+        if (token.fontStyle & FontStyle.Italic)
+            styles['font-style'] = 'italic';
+        if (token.fontStyle & FontStyle.Bold)
+            styles['font-weight'] = 'bold';
+        if (token.fontStyle & FontStyle.Underline)
+            styles['text-decoration'] = 'underline';
+    }
+    return styles;
+}
+function stringifyTokenStyle(token) {
+    return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(';');
+}
 
 // src/colors.ts
 var namedColors = [
@@ -430,7 +450,10 @@ function dimColor(color) {
     return color;
 }
 
-function codeToThemedTokens(internal, code, options = {}) {
+/**
+ * Code to tokens, with a simple theme.
+ */
+function codeToTokensBase(internal, code, options = {}) {
     const { lang = 'text', theme: themeName = internal.getLoadedThemes()[0], } = options;
     if (isPlainLang(lang) || isNoneTheme(themeName))
         return splitLines(code).map(line => [{ content: line[0], offset: line[1] }]);
@@ -570,7 +593,7 @@ function codeToTokensWithThemes(internal, code, options) {
     const themes = Object.entries(options.themes)
         .filter(i => i[1])
         .map(i => ({ color: i[0], theme: i[1] }));
-    const tokens = syncThemesTokenization(...themes.map(t => codeToThemedTokens(internal, code, {
+    const tokens = syncThemesTokenization(...themes.map(t => codeToTokensBase(internal, code, {
         ...options,
         theme: t.theme,
     })));
@@ -638,14 +661,12 @@ function syncThemesTokenization(...themes) {
     return outThemes;
 }
 
-function codeToHast(internal, code, options, transformerContext = {
-    meta: {},
-    options,
-    codeToHast: (_code, _options) => codeToHast(internal, _code, _options),
-}) {
-    let input = code;
-    for (const transformer of options.transformers || [])
-        input = transformer.preprocess?.call(transformerContext, input, options) || input;
+/**
+ * High-level code-to-tokens API.
+ *
+ * It will use `codeToTokensWithThemes` or `codeToTokensBase` based on the options.
+ */
+function codeToTokens(internal, code, options) {
     let bg;
     let fg;
     let tokens;
@@ -659,7 +680,7 @@ function codeToHast(internal, code, options, transformerContext = {
             .sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0);
         if (themes.length === 0)
             throw new Error('[shiki] `themes` option must not be empty');
-        const themeTokens = codeToTokensWithThemes(internal, input, options);
+        const themeTokens = codeToTokensWithThemes(internal, code, options);
         if (defaultColor && !themes.find(t => t.color === defaultColor))
             throw new Error(`[shiki] \`themes\` option must contain the defaultColor key \`${defaultColor}\``);
         const themeRegs = themes.map(t => internal.getTheme(t.theme));
@@ -672,7 +693,7 @@ function codeToHast(internal, code, options, transformerContext = {
         rootStyle = defaultColor ? undefined : [fg, bg].join(';');
     }
     else if ('theme' in options) {
-        tokens = codeToThemedTokens(internal, input, options);
+        tokens = codeToTokensBase(internal, code, options);
         const _theme = internal.getTheme(options.theme);
         bg = _theme.bg;
         fg = _theme.fg;
@@ -681,20 +702,13 @@ function codeToHast(internal, code, options, transformerContext = {
     else {
         throw new Error('[shiki] Invalid options, either `theme` or `themes` must be provided');
     }
-    const { mergeWhitespaces = true, } = options;
-    if (mergeWhitespaces === true)
-        tokens = mergeWhitespaceTokens(tokens);
-    else if (mergeWhitespaces === 'never')
-        tokens = splitWhitespaceTokens(tokens);
-    for (const transformer of options.transformers || [])
-        tokens = transformer.tokens?.call(transformerContext, tokens) || tokens;
-    return tokensToHast(tokens, {
-        ...options,
+    return {
+        tokens,
         fg,
         bg,
         themeName,
         rootStyle,
-    }, transformerContext);
+    };
 }
 function mergeToken(merged, variantsOrder, cssVariablePrefix, defaultColor) {
     const token = {
@@ -727,6 +741,32 @@ function mergeToken(merged, variantsOrder, cssVariablePrefix, defaultColor) {
         : Object.values(mergedStyles).join(';');
     return token;
 }
+
+function codeToHast(internal, code, options, transformerContext = {
+    meta: {},
+    options,
+    codeToHast: (_code, _options) => codeToHast(internal, _code, _options),
+    codeToTokens: (_code, _options) => codeToTokens(internal, _code, _options),
+}) {
+    let input = code;
+    for (const transformer of options.transformers || [])
+        input = transformer.preprocess?.call(transformerContext, input, options) || input;
+    let { tokens, fg, bg, themeName, rootStyle, } = codeToTokens(internal, input, options);
+    const { mergeWhitespaces = true, } = options;
+    if (mergeWhitespaces === true)
+        tokens = mergeWhitespaceTokens(tokens);
+    else if (mergeWhitespaces === 'never')
+        tokens = splitWhitespaceTokens(tokens);
+    for (const transformer of options.transformers || [])
+        tokens = transformer.tokens?.call(transformerContext, tokens) || tokens;
+    return tokensToHast(tokens, {
+        ...options,
+        fg,
+        bg,
+        themeName,
+        rootStyle,
+    }, transformerContext);
+}
 function tokensToHast(tokens, options, transformerContext) {
     const { transformers = [], } = options;
     const lines = [];
@@ -755,6 +795,7 @@ function tokensToHast(tokens, options, transformerContext) {
     const lineNodes = [];
     const context = {
         ...transformerContext,
+        addClassToHast,
         get tokens() {
             return tokens;
         },
@@ -815,25 +856,6 @@ function tokensToHast(tokens, options, transformerContext) {
         result = transformer?.root?.call(context, result) || result;
     return result;
 }
-function getTokenStyleObject(token) {
-    const styles = {};
-    if (token.color)
-        styles.color = token.color;
-    if (token.bgColor)
-        styles['background-color'] = token.bgColor;
-    if (token.fontStyle) {
-        if (token.fontStyle & FontStyle.Italic)
-            styles['font-style'] = 'italic';
-        if (token.fontStyle & FontStyle.Bold)
-            styles['font-weight'] = 'bold';
-        if (token.fontStyle & FontStyle.Underline)
-            styles['text-decoration'] = 'underline';
-    }
-    return styles;
-}
-function stringifyTokenStyle(token) {
-    return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(';');
-}
 function mergeWhitespaceTokens(tokens) {
     return tokens.map((line) => {
         const newLine = [];
@@ -4277,6 +4299,7 @@ function codeToHtml(internal, code, options) {
         meta: {},
         options,
         codeToHast: (_code, _options) => codeToHast(internal, _code, _options),
+        codeToTokens: (_code, _options) => codeToTokens(internal, _code, _options),
     };
     let result = toHtml(codeToHast(internal, code, options, context));
     for (const transformer of options.transformers || [])
@@ -5073,8 +5096,9 @@ async function getShikiInternal(options = {}) {
 async function getHighlighterCore(options = {}) {
     const internal = await getShikiInternal(options);
     return {
-        codeToThemedTokens: (code, options) => codeToThemedTokens(internal, code, options),
+        codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options),
         codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options),
+        codeToTokens: (code, options) => codeToTokens(internal, code, options),
         codeToHast: (code, options) => codeToHast(internal, code, options),
         codeToHtml: (code, options) => codeToHtml(internal, code, options),
         loadLanguage: internal.loadLanguage,
@@ -5085,6 +5109,8 @@ async function getHighlighterCore(options = {}) {
         getLoadedThemes: internal.getLoadedThemes,
         getLoadedLanguages: internal.getLoadedLanguages,
         getInternalContext: () => internal,
+        /** @deprecated */
+        codeToThemedTokens: (code, options) => codeToTokensBase(internal, code, options),
     };
 }
 
@@ -5175,6 +5201,20 @@ function createSingletonShorthands(getHighlighter) {
             });
             return shiki.codeToHast(code, options);
         },
+        async codeToTokens(code, options) {
+            const shiki = await _getHighlighter({
+                lang: options.lang,
+                theme: ('theme' in options ? [options.theme] : Object.values(options.themes)),
+            });
+            return shiki.codeToTokens(code, options);
+        },
+        async codeToTokensBase(code, options) {
+            const shiki = await _getHighlighter(options);
+            return shiki.codeToTokensBase(code, options);
+        },
+        /**
+         * @deprecated Use `codeToTokensBase` instead.
+         */
         async codeToThemedTokens(code, options) {
             const shiki = await _getHighlighter(options);
             return shiki.codeToThemedTokens(code, options);
@@ -5189,4 +5229,4 @@ function createSingletonShorthands(getHighlighter) {
     };
 }
 
-export { FontStyle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToThemedTokens, codeToTokensWithThemes, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, toHtml as hastToHtml, isNoneTheme, isPlainLang, isPlaintext, isSpecialLang, isSpecialTheme, loadWasm, normalizeTheme, setDefaultWasmLoader, splitLines, splitToken, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast };
+export { FontStyle, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createSingletonShorthands, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isPlaintext, isSpecialLang, isSpecialTheme, loadWasm, normalizeTheme, setDefaultWasmLoader, splitLines, splitToken, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast };

package/dist/textmate.d.mts

@@ -1,5 +1,5 @@
 import { F as FontStyle } from './chunk-types.mjs';
-export { I as IGrammar, a8 as IGrammarConfiguration, a5 as INITIAL, a9 as IOnigLib, n as IRawGrammar, a7 as IRawTheme, ab as IRawThemeSetting, a4 as Registry, aa as RegistryOptions, a6 as StateStack } from './chunk-types.mjs';
+export { I as IGrammar, aa as IGrammarConfiguration, a7 as INITIAL, ab as IOnigLib, q as IRawGrammar, a9 as IRawTheme, ad as IRawThemeSetting, a6 as Registry, ac as RegistryOptions, a8 as StateStack } from './chunk-types.mjs';
 import './chunk-index.mjs';
 
 declare const enum TemporaryStandardTokenType {

package/dist/types.d.mts

@@ -1,2 +1,2 @@
-export { A as AnsiLanguage, p as Awaitable, B as BundledHighlighterOptions, $ as BundledLanguageInfo, a0 as BundledThemeInfo, z as CodeOptionsMeta, x as CodeOptionsMultipleThemes, w as CodeOptionsSingleTheme, y as CodeOptionsThemes, C as CodeToHastOptions, v as CodeToHastOptionsCommon, c as CodeToThemedTokensOptions, e as CodeToTokensWithThemesOptions, Z as DynamicImportLanguageRegistration, _ as DynamicImportThemeRegistration, F as FontStyle, I as Grammar, a as HighlighterCoreOptions, H as HighlighterGeneric, k as HtmlRendererOptions, U as HtmlRendererOptionsCommon, I as IGrammar, n as IRawGrammar, L as LanguageInput, u as LanguageRegistration, M as MaybeArray, q as MaybeGetter, r as MaybeModule, P as PlainTextLanguage, n as RawGrammar, b as RequireKeys, t as ResolveBundleKey, g as ShikiInternal, Q as ShikiTransformer, O as ShikiTransformerContext, j as ShikiTransformerContextCommon, K as ShikiTransformerContextMeta, a1 as ShikijiTransformer, a2 as ShikijiTransformerContext, a3 as ShikijiTransformerContextCommon, o as SpecialLanguage, S as SpecialTheme, s as StringLiteralUnion, T as ThemeInput, J as ThemeRegistration, m as ThemeRegistrationAny, G as ThemeRegistrationRaw, h as ThemeRegistrationResolved, d as ThemedToken, W as ThemedTokenExplanation, V as ThemedTokenScopeExplanation, f as ThemedTokenWithVariants, X as TokenBase, Y as TokenStyles, i as TokenizeWithThemeOptions, D as TransformerOptions } from './chunk-types.mjs';
+export { A as AnsiLanguage, s as Awaitable, B as BundledHighlighterOptions, a1 as BundledLanguageInfo, a2 as BundledThemeInfo, J as CodeOptionsMeta, D as CodeOptionsMultipleThemes, z as CodeOptionsSingleTheme, G as CodeOptionsThemes, C as CodeToHastOptions, y as CodeToHastOptionsCommon, n as CodeToHastRenderOptions, X as CodeToHastRenderOptionsCommon, e as CodeToTokensBaseOptions, b as CodeToTokensOptions, g as CodeToTokensWithThemesOptions, $ as DynamicImportLanguageRegistration, a0 as DynamicImportThemeRegistration, F as FontStyle, I as Grammar, a as HighlighterCoreOptions, H as HighlighterGeneric, I as IGrammar, q as IRawGrammar, L as LanguageInput, x as LanguageRegistration, M as MaybeArray, t as MaybeGetter, u as MaybeModule, P as PlainTextLanguage, q as RawGrammar, d as RequireKeys, w as ResolveBundleKey, j as ShikiInternal, W as ShikiTransformer, V as ShikiTransformerContext, m as ShikiTransformerContextCommon, U as ShikiTransformerContextMeta, a3 as ShikijiTransformer, a4 as ShikijiTransformerContext, a5 as ShikijiTransformerContextCommon, r as SpecialLanguage, S as SpecialTheme, v as StringLiteralUnion, T as ThemeInput, Q as ThemeRegistration, p as ThemeRegistrationAny, O as ThemeRegistrationRaw, k as ThemeRegistrationResolved, f as ThemedToken, Z as ThemedTokenExplanation, Y as ThemedTokenScopeExplanation, h as ThemedTokenWithVariants, _ as TokenBase, i as TokenStyles, l as TokenizeWithThemeOptions, c as TokensResult, K as TransformerOptions } from './chunk-types.mjs';
 import './chunk-index.mjs';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.0.0-beta.1",
+  "version": "1.0.0-beta.2",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",