@shikijs/core 1.6.4 → 1.7.0

package/dist/chunk-tokens.d.mts

@@ -1005,6 +1005,11 @@ interface HighlighterCoreOptions {
      * Load wasm file from a custom path or using a custom function.
      */
     loadWasm?: LoadWasmOptions;
+    /**
+     * Emit console warnings to alert users of potential issues.
+     * @default true
+     */
+    warnings?: boolean;
 }
 interface BundledHighlighterOptions<L extends string, T extends string> {
     /**

package/dist/index.mjs

@@ -537,6 +537,26 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     let ruleStack = INITIAL;
     let actual = [];
     const final = [];
+    const themeSettingsSelectors = [];
+    if (options.includeExplanation) {
+        for (const setting of theme.settings) {
+            let selectors;
+            switch (typeof setting.scope) {
+                case 'string':
+                    selectors = setting.scope.split(/,/).map(scope => scope.trim());
+                    break;
+                case 'object':
+                    selectors = setting.scope;
+                    break;
+                default:
+                    continue;
+            }
+            themeSettingsSelectors.push({
+                settings: setting,
+                selectors: selectors.map(selector => selector.split(/ /)),
+            });
+        }
+    }
     for (let i = 0, len = lines.length; i < len; i++) {
         const [line, lineOffset] = lines[i];
         if (line === '') {
@@ -588,7 +608,7 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
                     offset += tokenWithScopesText.length;
                     token.explanation.push({
                         content: tokenWithScopesText,
-                        scopes: explainThemeScopes(theme, tokenWithScopes.scopes),
+                        scopes: explainThemeScopes(themeSettingsSelectors, tokenWithScopes.scopes),
                     });
                     tokensWithScopesIndex += 1;
                 }
@@ -601,31 +621,29 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
     }
     return final;
 }
-function explainThemeScopes(theme, scopes) {
+function explainThemeScopes(themeSelectors, scopes) {
     const result = [];
     for (let i = 0, len = scopes.length; i < len; i++) {
         const parentScopes = scopes.slice(0, i);
         const scope = scopes[i];
         result[i] = {
             scopeName: scope,
-            themeMatches: explainThemeScope(theme, scope, parentScopes),
+            themeMatches: explainThemeScope(themeSelectors, scope, parentScopes),
         };
     }
     return result;
 }
 function matchesOne(selector, scope) {
-    const selectorPrefix = `${selector}.`;
-    if (selector === scope || scope.substring(0, selectorPrefix.length) === selectorPrefix)
-        return true;
-    return false;
+    return selector === scope
+        || (scope.substring(0, selector.length) === selector && scope[selector.length] === '.');
 }
-function matches(selector, selectorParentScopes, scope, parentScopes) {
-    if (!matchesOne(selector, scope))
+function matches(selectors, scope, parentScopes) {
+    if (!matchesOne(selectors[selectors.length - 1], scope))
         return false;
-    let selectorParentIndex = selectorParentScopes.length - 1;
+    let selectorParentIndex = selectors.length - 2;
     let parentIndex = parentScopes.length - 1;
     while (selectorParentIndex >= 0 && parentIndex >= 0) {
-        if (matchesOne(selectorParentScopes[selectorParentIndex], parentScopes[parentIndex]))
+        if (matchesOne(selectors[selectorParentIndex], parentScopes[parentIndex]))
             selectorParentIndex -= 1;
         parentIndex -= 1;
     }
@@ -633,28 +651,13 @@ function matches(selector, selectorParentScopes, scope, parentScopes) {
         return true;
     return false;
 }
-function explainThemeScope(theme, scope, parentScopes) {
+function explainThemeScope(themeSettingsSelectors, scope, parentScopes) {
     const result = [];
-    let resultLen = 0;
-    for (let i = 0, len = theme.settings.length; i < len; i++) {
-        const setting = theme.settings[i];
-        let selectors;
-        if (typeof setting.scope === 'string')
-            selectors = setting.scope.split(/,/).map(scope => scope.trim());
-        else if (Array.isArray(setting.scope))
-            selectors = setting.scope;
-        else
-            continue;
-        for (let j = 0, lenJ = selectors.length; j < lenJ; j++) {
-            const rawSelector = selectors[j];
-            const rawSelectorPieces = rawSelector.split(/ /);
-            const selector = rawSelectorPieces[rawSelectorPieces.length - 1];
-            const selectorParentScopes = rawSelectorPieces.slice(0, rawSelectorPieces.length - 1);
-            if (matches(selector, selectorParentScopes, scope, parentScopes)) {
-                // match!
-                result[resultLen++] = setting;
-                // break the loop
-                j = lenJ;
+    for (const { selectors, settings } of themeSettingsSelectors) {
+        for (const selectorPieces of selectors) {
+            if (matches(selectorPieces, scope, parentScopes)) {
+                result.push(settings);
+                break; // continue to the next theme settings
             }
         }
     }
@@ -4648,6 +4651,50 @@ async function main(init) {
         }
         return false;
     }
+    const UTF8Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder('utf8') : undefined;
+    function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead = 1024) {
+        const endIdx = idx + maxBytesToRead;
+        let endPtr = idx;
+        while (heapOrArray[endPtr] && !(endPtr >= endIdx))
+            ++endPtr;
+        if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) {
+            return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr));
+        }
+        let str = '';
+        while (idx < endPtr) {
+            let u0 = heapOrArray[idx++];
+            if (!(u0 & 128)) {
+                str += String.fromCharCode(u0);
+                continue;
+            }
+            const u1 = heapOrArray[idx++] & 63;
+            if ((u0 & 224) === 192) {
+                str += String.fromCharCode(((u0 & 31) << 6) | u1);
+                continue;
+            }
+            const u2 = heapOrArray[idx++] & 63;
+            if ((u0 & 240) === 224) {
+                u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
+            }
+            else {
+                u0 = ((u0 & 7) << 18)
+                    | (u1 << 12)
+                    | (u2 << 6)
+                    | (heapOrArray[idx++] & 63);
+            }
+            if (u0 < 65536) {
+                str += String.fromCharCode(u0);
+            }
+            else {
+                const ch = u0 - 65536;
+                str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023));
+            }
+        }
+        return str;
+    }
+    function UTF8ToString(ptr, maxBytesToRead) {
+        return ptr ? UTF8ArrayToString(binding.HEAPU8, ptr, maxBytesToRead) : '';
+    }
     const asmLibraryArg = {
         emscripten_get_now: _emscripten_get_now,
         emscripten_memcpy_big: _emscripten_memcpy_big,
@@ -4663,6 +4710,7 @@ async function main(init) {
         wasmMemory = exports.memory;
         updateGlobalBufferAndViews(wasmMemory.buffer);
         Object.assign(binding, exports);
+        binding.UTF8ToString = UTF8ToString;
     }
     await createWasm();
     return binding;
@@ -5335,10 +5383,14 @@ let _defaultWasmLoader;
 function setDefaultWasmLoader(_loader) {
     _defaultWasmLoader = _loader;
 }
+let instancesCount = 0;
 /**
  * Get the minimal shiki context for rendering.
  */
 async function getShikiInternal(options = {}) {
+    instancesCount += 1;
+    if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0)
+        console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance.`);
     async function normalizeGetter(p) {
         return Promise.resolve(typeof p === 'function' ? p() : p).then(r => r.default || r);
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.6.4",
+  "version": "1.7.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",