@mirascript/monaco 0.1.80 → 0.1.83

package/src/basic/tokens-provider.ts

@@ -1,475 +1,136 @@
+import type { HighlighterCore, Grammar } from '@shikijs/core';
+import type { StateStack } from '@shikijs/vscode-textmate';
 import { languages, type IDisposable } from '../monaco-api.js';
-import {
-    REG_WHITESPACE,
-    REG_ORDINAL,
-    REG_OCT,
-    REG_BIN,
-    REG_HEX,
-    REG_NUMBER,
-    REG_IDENTIFIER,
-    MAX_VERBATIM_LENGTH,
-    CONSTANT_KEYWORDS,
-    CONTROL_KEYWORDS,
-    KEYWORDS,
-    NUMERIC_KEYWORDS,
-} from '../constants.js';
-import { DefaultVmContext } from '@mirascript/mirascript/subtle';
-import { isVmModule } from '@mirascript/mirascript';
+import { CONTRIBUTE_IDS } from '../contribute.js';
 
-const moduleNames = [...DefaultVmContext.keys()].filter(
-    (name) => DefaultVmContext.has(name) && isVmModule(DefaultVmContext.get(name)),
-);
+const TOKENIZE_MAX_LINE_LENGTH = 20000;
+const TOKENIZE_TIME_LIMIT = 500;
 
-/** 匹配 identifier */
-function identifierCases(
-    data?: Partial<languages.IExpandedMonarchLanguageAction>,
-    defaultToken = 'variable',
-): Record<string, languages.IExpandedMonarchLanguageAction> {
-    return {
-        '@numericKeywords': { ...data, token: `constant.numeric` },
-        '@constantKeywords': { ...data, token: `constant.language` },
-        '@controlKeywords': { ...data, token: `keyword.flow` },
-        '@keywords': { ...data, token: `keyword` },
-        '[@]+.*': { ...data, token: `variable.other.constant` },
-        '~it': { ...data, token: `variable.other.constant.emphasis` },
-        '@default': { ...data, token: defaultToken },
-    };
+/** Select the deepest scope that a native Monaco theme can style. */
+function tokenScope(scopes: string[]): string {
+    const inInterpolation = scopes.some((scope) => scope.startsWith('meta.interpolation.'));
+    let fallback = '';
+    const styledScopePrefixes = [
+        'invalid.',
+        'comment.',
+        'string.',
+        'keyword.',
+        'constant.',
+        'variable.',
+        'entity.',
+        'storage.',
+        'support.',
+        'markup.',
+    ];
+    for (let index = scopes.length - 1; index >= 0; index -= 1) {
+        const scope = scopes[index]!;
+        if (scope === 'source.mira' || scope === 'source.mira.doc' || scope === 'text.miratpl') continue;
+        if (scope.startsWith('meta.')) continue;
+        fallback ||= scope;
+        if (inInterpolation || styledScopePrefixes.some((prefix) => scope.startsWith(prefix))) return scope;
+    }
+    return fallback;
 }
 
-/** 生成 TokensProvider */
-function getTokensProvider(mode: string): languages.IMonarchLanguage {
-    return {
-        ignoreCase: false,
-        unicode: true,
-        includeLF: false,
-        brackets: [
-            { open: '{', close: '}', token: 'delimiter.curly' },
-            { open: '[', close: ']', token: 'delimiter.square' },
-            { open: '(', close: ')', token: 'delimiter.parenthesis' },
-        ],
-        defaultToken: 'invalid',
-
-        whitespace: REG_WHITESPACE,
-        identifier: REG_IDENTIFIER,
-        identifierNoAtOnly: /(?:(?:_+|\$+|\p{XID_Start})\p{XID_Continue}*|@+\p{XID_Continue}+)/u,
-
-        keywords: KEYWORDS,
-        controlKeywords: CONTROL_KEYWORDS,
-        constantKeywords: CONSTANT_KEYWORDS,
-        numericKeywords: NUMERIC_KEYWORDS,
-
-        start: mode === 'template' ? 'root_template' : mode === 'doc' ? 'root_doc' : 'root',
-        tokenPostfix: '.mirascript',
-        tokenizer: {
-            root: [
-                [/[[\](){}]/, '@brackets'],
-                // 用于修正关键字做为属性名时的高亮问题，由于与格式化字符串冲突，仅 root 规则启用，其余情况改由 semantic 高亮处理
-                [/(0|[1-9]\d*|@identifier)(@whitespace*)(\??:)(?!:)/, ['variable.other.property', '', 'delimiter']],
-                { include: '@common' },
-            ],
-            root_template: [
-                [/[^$]+/, 'string'],
-                [/(?=\$)/, '', '@string_interpolation.$S3'],
-                [/[$]/, 'string'],
-            ],
-            common: [
-                [
-                    /(mod)(@whitespace+)(@identifier)(?=$|@whitespace|[[({,;])/,
-                    ['keyword', '', { cases: identifierCases(undefined, 'entity.name.namespace') }],
-                ],
-                [
-                    /(fn)(@whitespace+)(@identifier)(?=$|@whitespace|[[({,;])/,
-                    ['keyword', '', { cases: identifierCases(undefined, 'entity.name.function') }],
-                ],
-                [
-                    /(for)(@whitespace+)(mut)(@whitespace+)(@identifier)(@whitespace+)(in)/,
-                    ['keyword.flow', '', 'keyword', '', { cases: identifierCases() }, '', 'keyword.flow'],
-                ],
-                [
-                    /(for)(@whitespace+)(@identifier)(@whitespace+)(in)/,
-                    ['keyword.flow', '', { cases: identifierCases() }, '', 'keyword.flow'],
-                ],
-                [
-                    /(\.)(@whitespace*)(\d+\b)/,
-                    [
-                        'delimiter',
-                        '',
-                        {
-                            cases: {
-                                [REG_ORDINAL.source]: 'variable',
-                                '@default': 'number.float',
-                            },
-                        },
-                    ],
-                ],
-                [String.raw`\b(${moduleNames.join('|')})(@whitespace*(?=!?\.))`, ['type', '']],
-                [
-                    /(\.)(@whitespace*)(@identifierNoAtOnly)(@whitespace*)(!?)(@whitespace*(?=\(|@*['"`]))/,
-                    ['delimiter', '', 'entity.name.function', '', 'delimiter', ''],
-                ],
-                [/(\.)(@whitespace*)(@identifier\b)/, ['delimiter', '', 'variable']],
-                // 不可变通过 semantic token 处理，避免在无 semantic token 支持的环境下高亮不一致，此处仅用于将非保留关键字识别为 identifier。
-                [/(let)(@whitespace+)(@identifier)(@whitespace+)(=)/, ['keyword', '', 'variable', '', 'delimiter']],
-                [
-                    /(type)(@whitespace*)(!)(@whitespace*)([(])/,
-                    ['entity.name.function', '', 'delimiter', '', '@brackets'],
-                ],
-                [/(type)(@whitespace*)([-+=/~?:;,.!@$%^&|*<>])/, ['variable', '', 'delimiter']],
-                [
-                    /(@identifierNoAtOnly)(@whitespace*)(!?)(@whitespace*(?=\(|@*['"`]))/,
-                    [
-                        {
-                            cases: identifierCases(undefined, `entity.name.function`),
-                        },
-                        '',
-                        'delimiter',
-                        '',
-                    ],
-                ],
-                { include: '@whitespace' },
-                { include: '@string' },
-                [/(@identifier)/, { cases: identifierCases() }],
-                [
-                    /0[xobXOB]\p{XID_Continue}*/u,
-                    {
-                        cases: {
-                            [REG_OCT.source]: 'number.octal',
-                            [REG_BIN.source]: 'number.binary',
-                            [REG_HEX.source]: 'number.hex',
-                            '@default': 'number.invalid',
-                        },
-                    },
-                ],
-                [
-                    REG_NUMBER,
-                    {
-                        cases: {
-                            [REG_ORDINAL.source]: 'number.ordinal',
-                            '@default': 'number.float',
-                        },
-                    },
-                ],
-                [/(\.\.|\?:|::|[-+=/~?:;,.!@$%^&|*<>])/, 'delimiter'],
-            ],
-            whitespace: [
-                [/(@whitespace)+/, ''],
-                [/\/\/.*$/, 'comment.line'],
-                [/\/\*{2}(?!\/)/, 'comment.doc', '@doc_comment'],
-                [/\/\*/, 'comment.block', '@block_comment'],
-            ],
-            format: [[/:(?!:)/, 'punctuation.format', '@format_string']],
-            format_string: [
-                [/\\./, 'string.escape.format'],
-                [/\(/, { token: 'string.format', next: '@format_string_inner' }],
-                [/\)/, { token: 'string.format', next: '@pop', goBack: 1 }],
-                [/\[/, { token: 'string.format', next: '@format_string_class' }],
-                [/[^()\\[]+/, 'string.format'],
-            ],
-            format_string_inner: [
-                [/\\./, 'string.escape.format'],
-                [/\(/, { token: 'string.format', next: '@push' }],
-                [/\)/, { token: 'string.format', next: '@pop' }],
-                [/\[/, { token: 'string.format', next: '@format_string_class' }],
-                [/[^()\\[\]]+/, 'string.format'],
-            ],
-            format_string_class: [
-                [/\\./, 'string.escape.format'],
-                [/\]/, { token: 'string.format', next: '@pop' }],
-                [/[^\\\]]+/, 'string.format'],
-            ],
-            string: [
-                [/["'`]/, { token: 'string.quote.open', next: '@string_normal.$#', bracket: '@open' }],
-                [
-                    /(@+)(["'`])/,
-                    { token: 'string.quote.open.$2$1.raw', next: '@string_verbatim.$2$1.$1', bracket: '@open' },
-                ],
-            ],
-            string_normal: [
-                [/[^'"`\\$]+/, 'string'],
-                { include: '@string_escape' },
-                [/(?=\$)/, '', '@string_interpolation.'],
-                [
-                    /['"`]/,
-                    {
-                        cases: {
-                            '$S2==$#': { token: 'string.quote.close', next: '@pop', bracket: '@close' },
-                            '@default': 'string',
-                        },
-                    },
-                ],
-            ],
-            string_verbatim: [
-                [/[^'"`$]+/, 'string'],
-                [/(?=\$)/, '', '@string_interpolation.$S3'],
-                [
-                    /(['"`]@+)/,
-                    {
-                        cases: {
-                            '$S2==$#': { token: 'string.quote.close.raw.$#', next: '@pop', bracket: '@close' },
-                            '@default': 'string',
-                        },
-                    },
-                ],
-                [/['"`$]/, 'string'],
-            ],
-            string_escape: [
-                [/\\([\\'"`$rntbfv0])/, 'string.escape'],
-                [/\\u\{([0-9a-fA-F]+)\}/, 'string.escape.unicode'],
-                [/\\x([0-9a-fA-F]{2})/, 'string.escape.ascii'],
-                [/\\./, { token: 'string.escape.invalid' }],
-            ],
-            ...Object.fromEntries(
-                Array.from({ length: MAX_VERBATIM_LENGTH }, (_, i) => {
-                    const dollarCount = i === 0 ? 1 : i;
-                    const dollarRegex = `\\\${${dollarCount}}`;
-                    return [
-                        `string_interpolation.${'@'.repeat(i)}`,
-                        [
-                            [
-                                `(${dollarRegex})(${REG_IDENTIFIER.source})`,
-                                ['punctuation.section.embedded', { cases: identifierCases({ next: '@pop' }) }],
-                            ],
-                            [
-                                String.raw`(${dollarRegex}\{)`,
-                                {
-                                    token: 'punctuation.section.embedded',
-                                    bracket: '@open',
-                                    next: '@braced',
-                                },
-                            ],
-                            [
-                                String.raw`(${dollarRegex}\()`,
-                                {
-                                    token: 'punctuation.section.embedded',
-                                    bracket: '@open',
-                                    next: '@parenthesized',
-                                },
-                            ],
-                            [`\\\${0,${dollarCount}}`, 'string', '@pop'],
-                            ['', '', '@pop'],
-                        ],
-                    ];
-                }),
-            ),
-            string_interpolation: [[/\$*/, 'string', '@pop']],
-
-            braced: [
-                [/\{/, { token: '@brackets', next: '@braced_inner' }],
-                [/\}/, { token: 'punctuation.section.embedded', bracket: '@close', next: '@pop' }],
-                [/\(/, { token: '@brackets', next: '@parenthesized_inner' }],
-                [/[[\])]/, '@brackets'],
-                { include: '@common' },
-            ],
-            braced_inner: [
-                [/\{/, { token: '@brackets', next: '@push' }],
-                [/\}/, { token: '@brackets', next: '@pop' }],
-                [/[[\]()]/, '@brackets'],
-                { include: '@common' },
-            ],
-            parenthesized: [
-                [/\(/, { token: '@brackets', next: '@parenthesized_inner' }],
-                [/\)/, { token: 'punctuation.section.embedded', bracket: '@close', next: '@pop' }],
-                [/\{/, { token: '@brackets', next: '@braced_inner' }],
-                [/[[\]}]/, '@brackets'],
-                { include: '@format' },
-                { include: '@common' },
-            ],
-            parenthesized_inner: [
-                [/\(/, { token: '@brackets', next: '@push' }],
-                [/\)/, { token: '@brackets', next: '@pop' }],
-                [/[[\]{}]/, '@brackets'],
-                { include: '@common' },
-            ],
+/** Shared instance of highlighter. */
+class HighlighterManager implements IDisposable {
+    private highlighterPromise: Promise<HighlighterCore> | null = null;
 
-            block_comment: [
-                [/\*\//, { token: 'comment.block', next: '@pop' }],
-                [/[^*]+/, { token: 'comment.block' }],
-                [/\*/, { token: 'comment.block' }],
-            ],
+    /**
+     * Get the shared highlighter instance.
+     */
+    private async getHighlighter(): Promise<HighlighterCore> {
+        if (this.highlighterPromise) return this.highlighterPromise;
 
-            doc_comment: [
-                [/\*\//, { token: 'comment.doc', next: '@pop' }],
-                [/^(\s*)\*(?!\/)/, { token: 'comment.doc' }],
-                [/\\\*(?!\/)/, { token: 'comment.doc.escape' }],
-                [/@(param|returns)/, { token: 'entity.name.tag.doc' }],
-                [/\*{2}(\S|\S.*?\S)\*{2}(?!\/)/, { token: 'comment.strong' }],
-                [/\*(\S|\S.*?\S)\*(?!\/)/, { token: 'comment.emphasis' }],
-                [/[^*@\\]+/, { token: 'comment.doc' }],
-                [/[*@\\]/, { token: 'comment.doc' }],
-            ],
+        const [
+            { createHighlighterCore },
+            { createOnigurumaEngine },
+            wasm,
+            { mirascript, mirascriptDoc, mirascriptTemplate },
+        ] = await Promise.all([
+            import('@shikijs/core'),
+            import('@shikijs/engine-oniguruma'),
+            import('@shikijs/engine-oniguruma/wasm-inlined'),
+            import('@mirascript/textmate'),
+        ]);
+        this.highlighterPromise = createHighlighterCore({
+            langs: [mirascript, mirascriptDoc, mirascriptTemplate],
+            themes: [],
+            engine: createOnigurumaEngine(wasm),
+        });
+        return this.highlighterPromise;
+    }
 
-            root_doc: [
-                // inline doc, start with `\0`
-                [/(?=^\0)/, { token: '', switchTo: '@inline_doc' }],
-                [/(?=.)/, { token: '', switchTo: '@doc_mode' }],
-            ],
-
-            inline_doc: [
-                [
-                    /(\0\(parameter(?: pattern)?\))(@whitespace+)(\.\.|)(mut)(@whitespace+)(@identifier)/,
-                    ['entity.name.label', '', 'delimiter', 'keyword.mut', '', 'variable.emphasis'],
-                ],
-                [
-                    /(\0\(parameter(?: pattern)?\))(@whitespace+)(\.\.|)(@identifier)/,
-                    ['entity.name.label', '', 'delimiter', 'variable.other.constant.emphasis'],
-                ],
-                [/(\0\([^)]+\))(@whitespace+)/, ['entity.name.label', '']],
-                [/\b(@identifier)(?=\s*=\s*mod\s+)/, 'entity.name.namespace'],
-                [/\b(@identifier)(?=\s*=)/, 'variable'],
-                { include: '@doc_mode' },
-            ],
-
-            doc_mode: [
-                [
-                    /(@identifier)(@whitespace*)(\??:)(@whitespace*)(\/\*@whitespace*<)(extern )((?:async )?function\*?)(>@whitespace*\*\/)/,
-                    [
-                        'entity.name.function.doc',
-                        '',
-                        'delimiter',
-                        '',
-                        'comment.doc',
-                        'type.doc',
-                        'keyword.javascript',
-                        'comment.doc',
-                    ],
-                ],
-                [
-                    /(@identifier)(@whitespace*)(\??:)(@whitespace*)(\/\*@whitespace*<)(extern )(class)(@whitespace*)([<>.\w]*)(>@whitespace*\*\/)/,
-                    [
-                        'type.doc',
-                        '',
-                        'delimiter',
-                        '',
-                        'comment.doc',
-                        'type.doc',
-                        'keyword.javascript',
-                        '',
-                        'type.javascript',
-                        'comment.doc',
-                    ],
-                ],
-                [
-                    /(@identifier)(@whitespace*)(\??:)(@whitespace*)(\/\*@whitespace*<)(extern )([\w]*)(>@whitespace*\*\/)/,
-                    [
-                        'variable.other.property.doc',
-                        '',
-                        'delimiter',
-                        '',
-                        'comment.doc',
-                        'type.doc',
-                        'type.javascript',
-                        'comment.doc',
-                    ],
-                ],
-                [
-                    /(@identifier)(@whitespace*)(\??:)(@whitespace*)(\/\*@whitespace*<)(function )([.\w]*)(>@whitespace*\*\/)/,
-                    [
-                        'entity.name.function.doc',
-                        '',
-                        'delimiter',
-                        '',
-                        'comment.doc',
-                        'type.doc',
-                        'entity.name.label',
-                        'comment.doc',
-                    ],
-                ],
-                [/(@identifier)(@whitespace*)(\??:)(@whitespace+)/, ['variable.other.property', '', 'delimiter', '']],
+    /** Get tokens provider factory of language */
+    getTokensProviderFactory(languageId: string): languages.TokensProviderFactory {
+        return {
+            create: async () => {
+                const highlighter = await this.getHighlighter();
+                const { INITIAL } = await import('@shikijs/vscode-textmate');
+                const grammar = highlighter.getLanguage(languageId);
+                return new TokensProvider(grammar, INITIAL);
+            },
+        };
+    }
+    /** @inheritdoc */
+    dispose(): void {
+        const promise = this.highlighterPromise;
+        this.highlighterPromise = null;
+        if (promise) {
+            promise
+                .then((highlighter) => highlighter.dispose())
+                .catch(() => {
+                    // eslint-disable-next-line no-console
+                    console.error('Failed to dispose highlighter');
+                });
+        }
+    }
+}
 
-                [
-                    /(\/\*@whitespace*<)(extern )((?:async )?function\*?)(>@whitespace*\*\/)/,
-                    ['comment.doc', 'type.doc', 'keyword.javascript', 'comment.doc'],
-                ],
-                [
-                    /(\/\*@whitespace*<)(extern )(class)(@whitespace*)([<>.\w]*)(>@whitespace*\*\/)/,
-                    ['comment.doc', 'type.doc', 'keyword.javascript', '', 'type.javascript', 'comment.doc'],
-                ],
-                [
-                    /(\/\*@whitespace*<)(extern )([\w]*)(\()(\d+)(\))(>@whitespace*\*\/)/,
-                    [
-                        'comment.doc',
-                        'type.doc',
-                        'type.javascript',
-                        'delimiter',
-                        'number.doc',
-                        'delimiter',
-                        'comment.doc',
-                    ],
-                ],
-                [
-                    /(\/\*@whitespace*<)(extern )([\w]*)([^>]*)(>@whitespace*\*\/)/,
-                    ['comment.doc', 'type.doc', 'type.javascript', '', 'comment.doc'],
-                ],
-                [
-                    /(\/\*@whitespace*<)(function )([.\w]*)(>@whitespace*\*\/)/,
-                    ['comment.doc', 'type.doc', 'entity.name.label', 'comment.doc'],
-                ],
-                [
-                    /(\/\*@whitespace*<)(\w+@whitespace*)([.\w]*)(>@whitespace*\*\/)/,
-                    ['comment.doc', 'type.doc', 'entity.name.label', 'comment.doc'],
-                ],
+/** A Monaco tokens provider that uses TextMate grammars. */
+class TokensProvider implements languages.TokensProvider {
+    constructor(
+        private readonly grammar: Grammar,
+        private readonly initialState: StateStack,
+    ) {}
+    /** @inheritdoc */
+    getInitialState(): StateStack {
+        return this.initialState;
+    }
+    /** @inheritdoc */
+    tokenize(line: string, state: StateStack): languages.ILineTokens {
+        if (line.length >= TOKENIZE_MAX_LINE_LENGTH) {
+            // eslint-disable-next-line no-console
+            console.warn(
+                `MiraScript TextMate tokenization skipped for line exceeding ${TOKENIZE_MAX_LINE_LENGTH} characters: ${line.slice(0, 100)}`,
+            );
+            return {
+                endState: state,
+                tokens: [{ startIndex: 0, scopes: '' }],
+            };
+        }
 
-                [
-                    /(let)(@whitespace+)(mut)(@whitespace+)(@identifier)/,
-                    [{ token: 'keyword.$1' }, '', 'keyword.mut', '', 'variable'],
-                ],
-                [/(let|const)(@whitespace+)(@identifier)/, [{ token: 'keyword.$1' }, '', 'variable.other.constant']],
-                [/(fn)(@whitespace+)(@identifier)$/, ['keyword.fn.doc', '', 'entity.name.function.doc']],
-                [
-                    /(fn)(@whitespace+)(@identifier)(\((?=(?:@identifier|@whitespace+|,|\.\.)+\)))/,
-                    [
-                        'keyword.fn.doc',
-                        '',
-                        'entity.name.function.doc',
-                        { token: '@brackets', next: '@type_doc_no_type' },
-                    ],
-                ],
-                [
-                    /(fn)(@whitespace+)(@identifier)(\()(\.\.)(\))/,
-                    ['keyword.fn.doc', '', 'entity.name.function.doc', '@brackets', 'delimiter', '@brackets'],
-                ],
-                [
-                    /(fn)(@whitespace+)(@identifier)/,
-                    ['keyword.fn.doc', '', { token: 'entity.name.function.doc', next: '@type_doc' }],
-                ],
-                [/[[\](){}]/, '@brackets'],
-                { include: '@common' },
-            ],
-            type_doc: [
-                [/;/, { token: 'delimiter', next: '@pop', goBack: 1 }],
-                [/(fn)(\()/, ['type', '@brackets']],
-                [/(type)(\()(@identifier)(\))/, ['type', '@brackets', 'variable.emphasis.doc', '@brackets']],
-                [
-                    /(@identifier)(\??:)(@whitespace*)(fn)(\()/,
-                    ['entity.name.function.emphasis.doc', 'delimiter', '', 'type', '@brackets'],
-                ],
-                [/(@identifier)(\??:)/, ['variable.emphasis', 'delimiter']],
-                [/@identifier/, 'type'],
-                [/</, { token: 'delimiter', next: '@type_doc' }],
-                [/>/, { token: 'delimiter', next: '@pop' }],
-                [/[&|.,:?<>]/, 'delimiter'],
-                [/->/, 'delimiter'],
-                [/[[\]()]/, '@brackets'],
-                { include: '@string' },
-                { include: '@whitespace' },
-            ],
-            type_doc_no_type: [
-                [/\)/, { token: '@brackets', next: '@pop' }],
-                [/@identifier/, 'variable.emphasis'],
-                [/[,.]/, 'delimiter'],
-                [/[[\]()]/, '@brackets'],
-                { include: '@string' },
-                { include: '@whitespace' },
-            ],
-        },
-    };
+        const result = this.grammar.tokenizeLine(line, state, TOKENIZE_TIME_LIMIT);
+        if (result.stoppedEarly) {
+            // eslint-disable-next-line no-console
+            console.warn(`MiraScript TextMate tokenization timed out: ${line.slice(0, 100)}`);
+        }
+        return {
+            endState: result.ruleStack,
+            tokens: result.tokens.map((token) => ({
+                startIndex: token.startIndex,
+                scopes: tokenScope(token.scopes),
+            })),
+        };
+    }
 }
 
-/** 注册 Mirascript 的 TokensProvider */
+/** Register TextMate-backed token providers without changing Monaco themes. */
 export function registerMiraScriptTokensProvider(): IDisposable[] {
-    return [
-        languages.setMonarchTokensProvider('mirascript', getTokensProvider('script')),
-        languages.setMonarchTokensProvider('mirascript-template', getTokensProvider('template')),
-        languages.setMonarchTokensProvider('mirascript-doc', getTokensProvider('doc')),
-    ];
+    const manager = new HighlighterManager();
+    const disposables = CONTRIBUTE_IDS.map((id) =>
+        languages.registerTokensProviderFactory(id, manager.getTokensProviderFactory(id)),
+    );
+    disposables.push(manager);
+    return disposables;
 }

package/src/contribute.ts

@@ -1,25 +1,25 @@
+import { mirascriptLanguage, mirascriptDocLanguage, mirascriptTemplateLanguage } from '@mirascript/textmate/language';
 import { languages } from './monaco-api.js';
 
+export const CONTRIBUTE_IDS = [mirascriptLanguage.name, mirascriptTemplateLanguage.name, mirascriptDocLanguage.name];
+
 /** 注册语言 */
 export function registerContribution(): void {
     languages.register({
-        id: 'mirascript',
+        id: mirascriptLanguage.name,
         extensions: ['.mira'],
-        aliases: ['MiraScript', 'mirascript', 'mira'],
+        aliases: mirascriptLanguage.aliases,
         mimetypes: ['text/x-mirascript'],
     });
 
     languages.register({
-        id: 'mirascript-template',
+        id: mirascriptTemplateLanguage.name,
         extensions: ['.miratpl'],
-        aliases: ['MiraScriptTemplate', 'mirascript-template', 'miratpl'],
+        aliases: mirascriptTemplateLanguage.aliases,
         mimetypes: ['text/x-mirascript-template'],
     });
 
     languages.register({
-        id: 'mirascript-doc',
-        extensions: [],
-        aliases: [],
-        mimetypes: ['text/x-mirascript-doc'],
+        id: mirascriptDocLanguage.name,
     });
 }

package/src/index.ts

@@ -20,6 +20,8 @@ export class MiraScriptMonacoLoader implements IDisposable {
 
         languages.onLanguageEncountered('mirascript-template', _loadBasicFeatures);
         languages.onLanguage('mirascript-template', _loadFullFeatures);
+
+        languages.onLanguageEncountered('mirascript-doc', _loadBasicFeatures);
     }
     features: LspFeaturesConfig = {};
     private _basicFeaturesLoaded = false;