@shikijs/core 1.16.0 → 1.16.2

package/dist/chunk-engines.d.mts

@@ -1,198 +1,4 @@
-interface IOnigCaptureIndex {
-    start: number;
-    end: number;
-    length: number;
-}
-interface IOnigMatch {
-    index: number;
-    captureIndices: IOnigCaptureIndex[];
-}
-declare const enum FindOption {
-    None = 0,
-    /**
-     * equivalent of ONIG_OPTION_NOT_BEGIN_STRING: (str) isn't considered as begin of string (* fail \A)
-     */
-    NotBeginString = 1,
-    /**
-     * equivalent of ONIG_OPTION_NOT_END_STRING: (end) isn't considered as end of string (* fail \z, \Z)
-     */
-    NotEndString = 2,
-    /**
-     * equivalent of ONIG_OPTION_NOT_BEGIN_POSITION: (start) isn't considered as start position of search (* fail \G)
-     */
-    NotBeginPosition = 4,
-    /**
-     * used for debugging purposes.
-     */
-    DebugCall = 8
-}
-interface OnigScanner {
-    findNextMatchSync(string: string | OnigString, startPosition: number, options: OrMask<FindOption>): IOnigMatch | null;
-    dispose?(): void;
-}
-interface OnigString {
-    readonly content: string;
-    dispose?(): void;
-}
-
-/**
- * A union of given const enum values.
-*/
-type OrMask<T extends number> = number;
-/**
- * Identifiers with a binary dot operator.
- * Examples: `baz` or `foo.bar`
-*/
-type ScopeName = string;
-/**
- * An expression language of ScopePathStr with a binary comma (to indicate alternatives) operator.
- * Examples: `foo.bar boo.baz,quick quack`
-*/
-type ScopePattern = string;
-/**
- * A TextMate theme.
- */
-interface IRawTheme {
-    readonly name?: string;
-    readonly settings: IRawThemeSetting[];
-}
-/**
- * A single theme setting.
- */
-interface IRawThemeSetting {
-    readonly name?: string;
-    readonly scope?: ScopePattern | ScopePattern[];
-    readonly settings: {
-        readonly fontStyle?: string;
-        readonly foreground?: string;
-        readonly background?: string;
-    };
-}
-
-declare const ruleIdSymbol: unique symbol;
-type RuleId = {
-    __brand: typeof ruleIdSymbol;
-};
-
-interface IRawGrammar extends ILocatable {
-    repository: IRawRepository;
-    readonly scopeName: ScopeName;
-    readonly patterns: IRawRule[];
-    readonly injections?: {
-        [expression: string]: IRawRule;
-    };
-    readonly injectionSelector?: string;
-    readonly fileTypes?: string[];
-    readonly name?: string;
-    readonly firstLineMatch?: string;
-}
-/**
- * Allowed values:
- * * Scope Name, e.g. `source.ts`
- * * Top level scope reference, e.g. `source.ts#entity.name.class`
- * * Relative scope reference, e.g. `#entity.name.class`
- * * self, e.g. `$self`
- * * base, e.g. `$base`
- */
-type IncludeString = string;
-type RegExpString = string;
-interface IRawRepositoryMap {
-    [name: string]: IRawRule;
-    $self: IRawRule;
-    $base: IRawRule;
-}
-type IRawRepository = IRawRepositoryMap & ILocatable;
-interface IRawRule extends ILocatable {
-    id?: RuleId;
-    readonly include?: IncludeString;
-    readonly name?: ScopeName;
-    readonly contentName?: ScopeName;
-    readonly match?: RegExpString;
-    readonly captures?: IRawCaptures;
-    readonly begin?: RegExpString;
-    readonly beginCaptures?: IRawCaptures;
-    readonly end?: RegExpString;
-    readonly endCaptures?: IRawCaptures;
-    readonly while?: RegExpString;
-    readonly whileCaptures?: IRawCaptures;
-    readonly patterns?: IRawRule[];
-    readonly repository?: IRawRepository;
-    readonly applyEndPatternLast?: boolean;
-}
-type IRawCaptures = IRawCapturesMap & ILocatable;
-interface IRawCapturesMap {
-    [captureId: string]: IRawRule;
-}
-interface ILocation {
-    readonly filename: string;
-    readonly line: number;
-    readonly char: number;
-}
-interface ILocatable {
-    readonly $vscodeTextmateLocation?: ILocation;
-}
-/**
- * A grammar
- */
-interface IGrammar {
-    /**
-     * Tokenize `lineText` using previous line state `prevState`.
-     */
-    tokenizeLine(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult;
-    /**
-     * Tokenize `lineText` using previous line state `prevState`.
-     * The result contains the tokens in binary format, resolved with the following information:
-     *  - language
-     *  - token type (regex, string, comment, other)
-     *  - font style
-     *  - foreground color
-     *  - background color
-     * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
-     */
-    tokenizeLine2(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult2;
-}
-interface ITokenizeLineResult {
-    readonly tokens: IToken[];
-    /**
-     * The `prevState` to be passed on to the next line tokenization.
-     */
-    readonly ruleStack: StateStack;
-    /**
-     * Did tokenization stop early due to reaching the time limit.
-     */
-    readonly stoppedEarly: boolean;
-}
-interface ITokenizeLineResult2 {
-    /**
-     * The tokens in binary format. Each token occupies two array indices. For token i:
-     *  - at offset 2*i => startIndex
-     *  - at offset 2*i + 1 => metadata
-     *
-     */
-    readonly tokens: Uint32Array;
-    /**
-     * The `prevState` to be passed on to the next line tokenization.
-     */
-    readonly ruleStack: StateStack;
-    /**
-     * Did tokenization stop early due to reaching the time limit.
-     */
-    readonly stoppedEarly: boolean;
-}
-interface IToken {
-    startIndex: number;
-    readonly endIndex: number;
-    readonly scopes: string[];
-}
-/**
- * **IMPORTANT** - Immutable!
- */
-interface StateStack {
-    _stackElementBrand: void;
-    readonly depth: number;
-    clone(): StateStack;
-    equals(other: StateStack): boolean;
-}
+import { OnigScanner, OnigString } from '@shikijs/vscode-textmate';
 
 type Awaitable<T> = T | Promise<T>;
 type MaybeGetter<T> = Awaitable<MaybeModule<T>> | (() => Awaitable<MaybeModule<T>>);
@@ -243,6 +49,12 @@ interface JavaScriptRegexEngineOptions {
      * Cache for regex patterns.
      */
     cache?: Map<string, RegExp | Error>;
+    /**
+     * Custom pattern to RegExp constructor.
+     *
+     * By default `oniguruma-to-js` is used.
+     */
+    regexConstructor?: (pattern: string) => RegExp;
 }
 
-export type { Awaitable as A, IGrammar as I, JavaScriptRegexEngineOptions as J, LoadWasmOptions as L, MaybeArray as M, OnigurumaLoadOptions as O, PatternScanner as P, RequireKeys as R, StringLiteralUnion as S, WebAssemblyInstantiator as W, RegexEngine as a, MaybeGetter as b, IRawGrammar as c, IRawTheme as d, IRawThemeSetting as e, MaybeModule as f, RegexEngineString as g, WebAssemblyInstance as h, LoadWasmOptionsPlain as i, StateStack as j };
+export type { Awaitable as A, JavaScriptRegexEngineOptions as J, LoadWasmOptions as L, MaybeArray as M, OnigurumaLoadOptions as O, PatternScanner as P, RequireKeys as R, StringLiteralUnion as S, WebAssemblyInstantiator as W, RegexEngine as a, MaybeGetter as b, MaybeModule as c, RegexEngineString as d, WebAssemblyInstance as e, LoadWasmOptionsPlain as f };

package/dist/index.d.mts

@@ -1,8 +1,10 @@
-import { HighlighterCoreOptions, HighlighterCore, BundledHighlighterOptions, HighlighterGeneric, CreatedBundledHighlighterOptions, LanguageInput, ThemeInput, CodeToHastOptions, CodeToTokensOptions, TokensResult, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, GrammarState, ShikiInternal, FontStyle, ThemeRegistrationAny, ThemeRegistrationResolved, TokenizeWithThemeOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, PlainTextLanguage, SpecialLanguage, SpecialTheme, TokenStyles, Position, ShikiTransformer } from './types.mjs';
-export { AnsiLanguage, BundledLanguageInfo, BundledThemeInfo, CodeOptionsMeta, CodeOptionsMultipleThemes, CodeOptionsSingleTheme, CodeOptionsThemes, CodeToHastOptionsCommon, CodeToHastRenderOptionsCommon, DecorationItem, DecorationOptions, DecorationTransformType, DynamicImportLanguageRegistration, DynamicImportThemeRegistration, Grammar, LanguageRegistration, Offset, OffsetOrPosition, ResolveBundleKey, ResolvedDecorationItem, ResolvedPosition, ShikiTransformerContext, ShikiTransformerContextMeta, ThemeRegistration, ThemeRegistrationRaw, ThemedTokenExplanation, ThemedTokenScopeExplanation, TokenBase, TransformerOptions } from './types.mjs';
-import { R as RequireKeys, L as LoadWasmOptions, a as RegexEngine, J as JavaScriptRegexEngineOptions, I as IGrammar, M as MaybeArray, b as MaybeGetter } from './chunk-engines.mjs';
-export { A as Awaitable, i as LoadWasmOptionsPlain, f as MaybeModule, O as OnigurumaLoadOptions, P as PatternScanner, c as RawGrammar, d as RawTheme, e as RawThemeSetting, g as RegexEngineString, S as StringLiteralUnion, h as WebAssemblyInstance, W as WebAssemblyInstantiator } from './chunk-engines.mjs';
+import { HighlighterCoreOptions, HighlighterCore, BundledHighlighterOptions, HighlighterGeneric, CreatedBundledHighlighterOptions, LanguageInput, ThemeInput, CodeToHastOptions, CodeToTokensOptions, TokensResult, CodeToTokensBaseOptions, ThemedToken, CodeToTokensWithThemesOptions, ThemedTokenWithVariants, GrammarState, ShikiInternal, ThemeRegistrationAny, ThemeRegistrationResolved, TokenizeWithThemeOptions, ShikiTransformerContextCommon, CodeToHastRenderOptions, ShikiTransformerContextSource, PlainTextLanguage, SpecialLanguage, SpecialTheme, TokenStyles, Position, ShikiTransformer } from './types.mjs';
+export { AnsiLanguage, BundledLanguageInfo, BundledThemeInfo, CodeOptionsMeta, CodeOptionsMultipleThemes, CodeOptionsSingleTheme, CodeOptionsThemes, CodeToHastOptionsCommon, CodeToHastRenderOptionsCommon, DecorationItem, DecorationOptions, DecorationTransformType, DynamicImportLanguageRegistration, DynamicImportThemeRegistration, FontStyle, Grammar, LanguageRegistration, Offset, OffsetOrPosition, ResolveBundleKey, ResolvedDecorationItem, ResolvedPosition, ShikiTransformerContext, ShikiTransformerContextMeta, ThemeRegistration, ThemeRegistrationRaw, ThemedTokenExplanation, ThemedTokenScopeExplanation, TokenBase, TransformerOptions } from './types.mjs';
+import { R as RequireKeys, L as LoadWasmOptions, a as RegexEngine, J as JavaScriptRegexEngineOptions, M as MaybeArray, b as MaybeGetter } from './chunk-engines.mjs';
+export { A as Awaitable, f as LoadWasmOptionsPlain, c as MaybeModule, O as OnigurumaLoadOptions, P as PatternScanner, d as RegexEngineString, S as StringLiteralUnion, e as WebAssemblyInstance, W as WebAssemblyInstantiator } from './chunk-engines.mjs';
 import { Root, RootContent, Nodes, Element } from 'hast';
+import { IGrammar } from '@shikijs/vscode-textmate';
+export { IRawGrammar as RawGrammar, IRawTheme as RawTheme, IRawThemeSetting as RawThemeSetting, EncodedTokenMetadata as StackElementMetadata } from '@shikijs/vscode-textmate';
 
 /**
  * Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -129,6 +131,10 @@ declare function loadWasm(options: LoadWasmOptions): Promise<void>;
 
 declare function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise<RegexEngine>;
 
+/**
+ * The default RegExp constructor for JavaScript regex engine.
+ */
+declare function defaultJavaScriptRegexConstructor(pattern: string): RegExp;
 /**
  * Use the modern JavaScript RegExp engine to implement the OnigScanner.
  *
@@ -140,24 +146,6 @@ declare function createWasmOnigEngine(options?: LoadWasmOptions | null): Promise
  */
 declare function createJavaScriptRegexEngine(options?: JavaScriptRegexEngineOptions): RegexEngine;
 
-declare const enum TemporaryStandardTokenType {
-    Other = 0,
-    Comment = 1,
-    String = 2,
-    RegEx = 4,
-    MetaEmbedded = 8
-}
-declare class StackElementMetadata {
-    static toBinaryStr(metadata: number): string;
-    static getLanguageId(metadata: number): number;
-    static getTokenType(metadata: number): number;
-    static getFontStyle(metadata: number): number;
-    static getForeground(metadata: number): number;
-    static getBackground(metadata: number): number;
-    static containsBalancedBrackets(metadata: number): boolean;
-    static set(metadata: number, languageId: number, tokenType: TemporaryStandardTokenType, fontStyle: FontStyle, foreground: number, background: number): number;
-}
-
 /**
  * Normalize a textmate theme to shiki theme
  */
@@ -486,4 +474,4 @@ declare class ShikiError extends Error {
     constructor(message: string);
 }
 
-export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, CreatedBundledHighlighterOptions, FontStyle, GrammarState, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, JavaScriptRegexEngineOptions, LanguageInput, LoadWasmOptions, MaybeArray, MaybeGetter, PlainTextLanguage, Position, RegexEngine, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, StackElementMetadata, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };
+export { BundledHighlighterOptions, CodeToHastOptions, CodeToHastRenderOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, type CreateHighlighterFactory, CreatedBundledHighlighterOptions, GrammarState, HighlighterCore, HighlighterCoreOptions, HighlighterGeneric, JavaScriptRegexEngineOptions, LanguageInput, LoadWasmOptions, MaybeArray, MaybeGetter, PlainTextLanguage, Position, RegexEngine, RequireKeys, ShikiError, ShikiInternal, ShikiTransformer, ShikiTransformerContextCommon, ShikiTransformerContextSource, type ShorthandsBundle, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved, ThemedToken, ThemedTokenWithVariants, TokenStyles, TokenizeWithThemeOptions, TokensResult, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createHighlighterCore, createHighlighterCoreSync, createJavaScriptRegexEngine, createPositionConverter, createShikiInternal, createShikiInternalSync, createSingletonShorthands, createWasmOnigEngine, createdBundledHighlighter, defaultJavaScriptRegexConstructor, getHighlighterCore, getShikiInternal, getSingletonHighlighterCore, getTokenStyleObject, toHtml as hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, loadWasm, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, setDefaultWasmLoader, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };