@shikijs/core 1.6.5 → 1.8.0

package/dist/chunk-tokens.d.mts

@@ -1005,6 +1005,11 @@ interface HighlighterCoreOptions {
      * Load wasm file from a custom path or using a custom function.
      */
     loadWasm?: LoadWasmOptions;
+    /**
+     * Emit console warnings to alert users of potential issues.
+     * @default true
+     */
+    warnings?: boolean;
 }
 interface BundledHighlighterOptions<L extends string, T extends string> {
     /**

package/dist/index.mjs

@@ -4651,6 +4651,50 @@ async function main(init) {
         }
         return false;
     }
+    const UTF8Decoder = typeof TextDecoder != 'undefined' ? new TextDecoder('utf8') : undefined;
+    function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead = 1024) {
+        const endIdx = idx + maxBytesToRead;
+        let endPtr = idx;
+        while (heapOrArray[endPtr] && !(endPtr >= endIdx))
+            ++endPtr;
+        if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) {
+            return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr));
+        }
+        let str = '';
+        while (idx < endPtr) {
+            let u0 = heapOrArray[idx++];
+            if (!(u0 & 128)) {
+                str += String.fromCharCode(u0);
+                continue;
+            }
+            const u1 = heapOrArray[idx++] & 63;
+            if ((u0 & 224) === 192) {
+                str += String.fromCharCode(((u0 & 31) << 6) | u1);
+                continue;
+            }
+            const u2 = heapOrArray[idx++] & 63;
+            if ((u0 & 240) === 224) {
+                u0 = ((u0 & 15) << 12) | (u1 << 6) | u2;
+            }
+            else {
+                u0 = ((u0 & 7) << 18)
+                    | (u1 << 12)
+                    | (u2 << 6)
+                    | (heapOrArray[idx++] & 63);
+            }
+            if (u0 < 65536) {
+                str += String.fromCharCode(u0);
+            }
+            else {
+                const ch = u0 - 65536;
+                str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023));
+            }
+        }
+        return str;
+    }
+    function UTF8ToString(ptr, maxBytesToRead) {
+        return ptr ? UTF8ArrayToString(binding.HEAPU8, ptr, maxBytesToRead) : '';
+    }
     const asmLibraryArg = {
         emscripten_get_now: _emscripten_get_now,
         emscripten_memcpy_big: _emscripten_memcpy_big,
@@ -4666,6 +4710,7 @@ async function main(init) {
         wasmMemory = exports.memory;
         updateGlobalBufferAndViews(wasmMemory.buffer);
         Object.assign(binding, exports);
+        binding.UTF8ToString = UTF8ToString;
     }
     await createWasm();
     return binding;
@@ -5154,9 +5199,9 @@ class Registry extends Registry$1 {
     _themes;
     _langs;
     _alias;
-    _resolvedThemes = {};
-    _resolvedGrammars = {};
-    _langMap = {};
+    _resolvedThemes = new Map();
+    _resolvedGrammars = new Map();
+    _langMap = new Map();
     _langGraph = new Map();
     _textmateThemeCache = new WeakMap();
     _loadedThemesCache = null;
@@ -5172,14 +5217,14 @@ class Registry extends Registry$1 {
     }
     getTheme(theme) {
         if (typeof theme === 'string')
-            return this._resolvedThemes[theme];
+            return this._resolvedThemes.get(theme);
         else
             return this.loadTheme(theme);
     }
     loadTheme(theme) {
         const _theme = normalizeTheme(theme);
         if (_theme.name) {
-            this._resolvedThemes[_theme.name] = _theme;
+            this._resolvedThemes.set(_theme.name, _theme);
             // Reset cache
             this._loadedThemesCache = null;
         }
@@ -5187,7 +5232,7 @@ class Registry extends Registry$1 {
     }
     getLoadedThemes() {
         if (!this._loadedThemesCache)
-            this._loadedThemesCache = Object.keys(this._resolvedThemes);
+            this._loadedThemesCache = [...this._resolvedThemes.keys()];
         return this._loadedThemesCache;
     }
     // Override and re-implement this method to cache the textmate themes as `TextMateTheme.createFromRawTheme`
@@ -5214,12 +5259,13 @@ class Registry extends Registry$1 {
                 resolved.add(name);
             }
         }
-        return this._resolvedGrammars[name];
+        return this._resolvedGrammars.get(name);
     }
     async loadLanguage(lang) {
         if (this.getGrammar(lang.name))
             return;
-        const embeddedLazilyBy = new Set(Object.values(this._langMap).filter(i => i.embeddedLangsLazy?.includes(lang.name)));
+        const embeddedLazilyBy = new Set([...this._langMap.values()]
+            .filter(i => i.embeddedLangsLazy?.includes(lang.name)));
         this._resolver.addLanguage(lang);
         const grammarConfig = {
             balancedBracketSelectors: lang.balancedBracketSelectors || ['*'],
@@ -5228,7 +5274,7 @@ class Registry extends Registry$1 {
         // @ts-expect-error Private members, set this to override the previous grammar (that can be a stub)
         this._syncRegistry._rawGrammars.set(lang.scopeName, lang);
         const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig);
-        this._resolvedGrammars[lang.name] = g;
+        this._resolvedGrammars.set(lang.name, g);
         if (lang.aliases) {
             lang.aliases.forEach((alias) => {
                 this._alias[alias] = lang.name;
@@ -5239,14 +5285,14 @@ class Registry extends Registry$1 {
         // If there is a language that embeds this language lazily, we need to reload it
         if (embeddedLazilyBy.size) {
             for (const e of embeddedLazilyBy) {
-                delete this._resolvedGrammars[e.name];
+                this._resolvedGrammars.delete(e.name);
                 // Reset cache
                 this._loadedLanguagesCache = null;
                 // @ts-expect-error clear cache
                 this._syncRegistry?._injectionGrammars?.delete(e.scopeName);
                 // @ts-expect-error clear cache
                 this._syncRegistry?._grammars?.delete(e.scopeName);
-                await this.loadLanguage(this._langMap[e.name]);
+                await this.loadLanguage(this._langMap.get(e.name));
             }
         }
     }
@@ -5254,6 +5300,14 @@ class Registry extends Registry$1 {
         this._themes.map(t => this.loadTheme(t));
         await this.loadLanguages(this._langs);
     }
+    dispose() {
+        super.dispose();
+        this._resolvedThemes.clear();
+        this._resolvedGrammars.clear();
+        this._langMap.clear();
+        this._langGraph.clear();
+        this._loadedThemesCache = null;
+    }
     async loadLanguages(langs) {
         for (const lang of langs)
             this.resolveEmbeddedLanguages(lang);
@@ -5271,16 +5325,19 @@ class Registry extends Registry$1 {
             await this.loadLanguage(lang);
     }
     getLoadedLanguages() {
-        if (!this._loadedLanguagesCache)
-            this._loadedLanguagesCache = Object.keys({ ...this._resolvedGrammars, ...this._alias });
+        if (!this._loadedLanguagesCache) {
+            this._loadedLanguagesCache = [
+                ...new Set([...this._resolvedGrammars.keys(), ...Object.keys(this._alias)]),
+            ];
+        }
         return this._loadedLanguagesCache;
     }
     resolveEmbeddedLanguages(lang) {
-        this._langMap[lang.name] = lang;
+        this._langMap.set(lang.name, lang);
         this._langGraph.set(lang.name, lang);
         if (lang.embeddedLangs) {
             for (const embeddedLang of lang.embeddedLangs)
-                this._langGraph.set(embeddedLang, this._langMap[embeddedLang]);
+                this._langGraph.set(embeddedLang, this._langMap.get(embeddedLang));
         }
     }
 }
@@ -5338,10 +5395,15 @@ let _defaultWasmLoader;
 function setDefaultWasmLoader(_loader) {
     _defaultWasmLoader = _loader;
 }
+let instancesCount = 0;
 /**
  * Get the minimal shiki context for rendering.
  */
 async function getShikiInternal(options = {}) {
+    instancesCount += 1;
+    if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0)
+        console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance; Or call \`highlighter.dispose()\` to release unused instances.`);
+    let isDisposed = false;
     async function normalizeGetter(p) {
         return Promise.resolve(typeof p === 'function' ? p() : p).then(r => r.default || r);
     }
@@ -5368,6 +5430,7 @@ async function getShikiInternal(options = {}) {
     await _registry.init();
     let _lastTheme;
     function getLanguage(name) {
+        ensureNotDisposed();
         const _lang = _registry.getGrammar(typeof name === 'string' ? name : name.name);
         if (!_lang)
             throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`);
@@ -5376,12 +5439,14 @@ async function getShikiInternal(options = {}) {
     function getTheme(name) {
         if (name === 'none')
             return { bg: '', fg: '', name: 'none', settings: [], type: 'dark' };
+        ensureNotDisposed();
         const _theme = _registry.getTheme(name);
         if (!_theme)
             throw new ShikiError(`Theme \`${name}\` not found, you may need to load it first`);
         return _theme;
     }
     function setTheme(name) {
+        ensureNotDisposed();
         const theme = getTheme(name);
         if (_lastTheme !== name) {
             _registry.setTheme(theme);
@@ -5394,19 +5459,34 @@ async function getShikiInternal(options = {}) {
         };
     }
     function getLoadedThemes() {
+        ensureNotDisposed();
         return _registry.getLoadedThemes();
     }
     function getLoadedLanguages() {
+        ensureNotDisposed();
         return _registry.getLoadedLanguages();
     }
     async function loadLanguage(...langs) {
+        ensureNotDisposed();
         await _registry.loadLanguages(await resolveLangs(langs));
     }
     async function loadTheme(...themes) {
+        ensureNotDisposed();
         await Promise.all(themes.map(async (theme) => isSpecialTheme(theme)
             ? null
             : _registry.loadTheme(await normalizeGetter(theme))));
     }
+    function ensureNotDisposed() {
+        if (isDisposed)
+            throw new ShikiError('Shiki instance has been disposed');
+    }
+    function dispose() {
+        if (isDisposed)
+            return;
+        isDisposed = true;
+        _registry.dispose();
+        instancesCount -= 1;
+    }
     return {
         setTheme,
         getTheme,
@@ -5415,6 +5495,8 @@ async function getShikiInternal(options = {}) {
         getLoadedLanguages,
         loadLanguage,
         loadTheme,
+        dispose,
+        [Symbol.dispose]: dispose,
     };
 }
 

package/dist/types.d.mts

@@ -42,6 +42,14 @@ interface ShikiInternal<BundledLangKeys extends string = never, BundledThemeKeys
      * Special-handled themes like `none` are not included.
      */
     getLoadedThemes: () => string[];
+    /**
+     * Dispose the internal registry and release resources
+     */
+    dispose: () => void;
+    /**
+     * Dispose the internal registry and release resources
+     */
+    [Symbol.dispose]: () => void;
 }
 /**
  * Generic instance interface of Shiki

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shikijs/core",
   "type": "module",
-  "version": "1.6.5",
+  "version": "1.8.0",
   "description": "Core of Shiki",
   "author": "Pine Wu <octref@gmail.com>; Anthony Fu <anthonyfu117@hotmail.com>",
   "license": "MIT",