@shikijs/core 1.1.2 → 1.1.4

package/dist/chunk-tokens.d.mts

@@ -999,7 +999,7 @@ interface CodeOptionsMultipleThemes<Themes extends string = string> {
 }
 type CodeOptionsThemes<Themes extends string = string> = CodeOptionsSingleTheme<Themes> | CodeOptionsMultipleThemes<Themes>;
 type CodeToHastOptions<Languages extends string = string, Themes extends string = string> = CodeToHastOptionsCommon<Languages> & CodeOptionsThemes<Themes> & CodeOptionsMeta;
-interface CodeToHastOptionsCommon<Languages extends string = string> extends TransformerOptions, DecorationOptions, Pick<TokenizeWithThemeOptions, 'colorReplacements'> {
+interface CodeToHastOptionsCommon<Languages extends string = string> extends TransformerOptions, DecorationOptions, Pick<TokenizeWithThemeOptions, 'colorReplacements' | 'tokenizeMaxLineLength' | 'tokenizeTimeLimit'> {
     lang: StringLiteralUnion<Languages | SpecialLanguage>;
     /**
      * Merge whitespace tokens to saving extra `<span>`.
@@ -1181,6 +1181,18 @@ interface TokenizeWithThemeOptions {
      * This will be merged with theme's `colorReplacements` if any.
      */
     colorReplacements?: Record<string, string>;
+    /**
+     * Lines above this length will not be tokenized for performance reasons.
+     *
+     * @default 0 (no limit)
+     */
+    tokenizeMaxLineLength?: number;
+    /**
+     * Time limit in milliseconds for tokenizing a single line.
+     *
+     * @default 500 (0.5s)
+     */
+    tokenizeTimeLimit?: number;
 }
 /**
  * Result of `codeToTokens`, an object with 2D array of tokens and meta info like background and foreground color.

package/dist/index.mjs

@@ -524,6 +524,7 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
         ...theme.colorReplacements,
         ...options?.colorReplacements,
     };
+    const { tokenizeMaxLineLength = 0, tokenizeTimeLimit = 500, } = options;
     const lines = splitLines(code);
     let ruleStack = INITIAL;
     let actual = [];
@@ -535,6 +536,17 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
             final.push([]);
             continue;
         }
+        // Do not attempt to tokenize if the line length is longer than the `tokenizationMaxLineLength`
+        if (tokenizeMaxLineLength > 0 && line.length >= tokenizeMaxLineLength) {
+            actual = [];
+            final.push([{
+                    content: line,
+                    offset: lineOffset,
+                    color: '',
+                    fontStyle: 0,
+                }]);
+            continue;
+        }
         let resultWithScopes;
         let tokensWithScopes;
         let tokensWithScopesIndex;
@@ -543,7 +555,7 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
             tokensWithScopes = resultWithScopes.tokens;
             tokensWithScopesIndex = 0;
         }
-        const result = grammar.tokenizeLine2(line, ruleStack);
+        const result = grammar.tokenizeLine2(line, ruleStack, tokenizeTimeLimit);
         const tokensLength = result.tokens.length / 2;
         for (let j = 0; j < tokensLength; j++) {
             const startIndex = result.tokens[2 * j];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.1.2",
+  "version": "1.1.4",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",