@codingame/monaco-vscode-api 1.78.4 → 1.78.5-next.0

package/dist/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js

@@ -0,0 +1,422 @@
+import { __decorate, __param } from '../../../../../../node_modules/tslib/tslib.es6.js';
+import * as dom from 'monaco-editor/esm/vs/base/browser/dom.js';
+import { equals } from 'monaco-editor/esm/vs/base/common/arrays.js';
+import { Color } from 'monaco-editor/esm/vs/base/common/color.js';
+import { onUnexpectedError } from 'monaco-editor/esm/vs/base/common/errors.js';
+import { Disposable, DisposableStore } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
+import { FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'monaco-editor/esm/vs/base/common/network.js';
+import { observableFromEvent } from 'monaco-editor/esm/vs/base/common/observable.js';
+import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
+import * as resources from 'monaco-editor/esm/vs/base/common/resources.js';
+import * as types from 'monaco-editor/esm/vs/base/common/types.js';
+import { TokenizationRegistry, LazyTokenizationSupport } from 'monaco-editor/esm/vs/editor/common/languages.js';
+import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
+import { generateTokensCSSForColorMap } from 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js';
+import * as nls from 'monaco-editor/esm/vs/nls.js';
+import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
+import { IExtensionResourceLoaderService } from '../../../../platform/extensionResourceLoader/common/extensionResourceLoader.js';
+import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
+import { ILogService } from 'monaco-editor/esm/vs/platform/log/common/log.js';
+import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
+import { IProgressService } from 'monaco-editor/esm/vs/platform/progress/common/progress.js';
+import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js';
+import { TextMateTokenizationSupport } from './tokenizationSupport/textMateTokenizationSupport.js';
+import { TokenizationSupportWithLineLimit } from './tokenizationSupport/tokenizationSupportWithLineLimit.js';
+import { TextMateWorkerHost } from './workerHost/textMateWorkerHost.js';
+import { TMGrammarFactory, missingTMGrammarErrorMessage } from '../common/TMGrammarFactory.js';
+import { grammarsExtPoint } from '../common/TMGrammars.js';
+import { IWorkbenchThemeService } from '../../themes/common/workbenchThemeService.js';
+let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
+    constructor(_languageService, _themeService, _extensionResourceLoaderService, _notificationService, _logService, _configurationService, _progressService, _environmentService, _instantiationService) {
+        super();
+        this._languageService = _languageService;
+        this._themeService = _themeService;
+        this._extensionResourceLoaderService = _extensionResourceLoaderService;
+        this._notificationService = _notificationService;
+        this._logService = _logService;
+        this._configurationService = _configurationService;
+        this._progressService = _progressService;
+        this._environmentService = _environmentService;
+        this._instantiationService = _instantiationService;
+        this._createdModes = [];
+        this._encounteredLanguages = [];
+        this._debugMode = false;
+        this._debugModePrintFunc = () => { };
+        this._grammarDefinitions = null;
+        this._grammarFactory = null;
+        this._tokenizersRegistrations = ( new DisposableStore());
+        this._currentTheme = null;
+        this._currentTokenColorMap = null;
+        this._workerHost = this._instantiationService.createInstance(TextMateWorkerHost);
+        this._vscodeOniguruma = null;
+        this._styleElement = dom.createStyleSheet();
+        this._styleElement.className = 'vscode-tokens-styles';
+        grammarsExtPoint.setHandler((extensions) => this.handleGrammarsExtPoint(extensions));
+        this._updateTheme(this._themeService.getColorTheme(), true);
+        this._register(this._themeService.onDidColorThemeChange(() => {
+            this._updateTheme(this._themeService.getColorTheme(), false);
+        }));
+        this._languageService.onDidRequestRichLanguageFeatures((languageId) => {
+            this._createdModes.push(languageId);
+        });
+    }
+    handleGrammarsExtPoint(extensions) {
+        this._grammarDefinitions = null;
+        if (this._grammarFactory) {
+            this._grammarFactory.dispose();
+            this._grammarFactory = null;
+        }
+        this._tokenizersRegistrations.clear();
+        this._grammarDefinitions = [];
+        for (const extension of extensions) {
+            const grammars = extension.value;
+            for (const grammar of grammars) {
+                const def = this.createValidGrammarDefinition(extension, grammar);
+                if (def) {
+                    this._grammarDefinitions.push(def);
+                    if (def.language) {
+                        const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this.createTokenizationSupport(def.language)));
+                        this._tokenizersRegistrations.add(lazyTokenizationSupport);
+                        this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(def.language, lazyTokenizationSupport));
+                    }
+                }
+            }
+        }
+        this._workerHost.setGrammarDefinitions(this._grammarDefinitions);
+        for (const createdMode of this._createdModes) {
+            TokenizationRegistry.getOrCreate(createdMode);
+        }
+    }
+    createValidGrammarDefinition(extension, grammar) {
+        if (!validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector, this._languageService)) {
+            return null;
+        }
+        const grammarLocation = resources.joinPath(extension.description.extensionLocation, grammar.path);
+        const embeddedLanguages = Object.create(null);
+        if (grammar.embeddedLanguages) {
+            const scopes = ( Object.keys(grammar.embeddedLanguages));
+            for (let i = 0, len = scopes.length; i < len; i++) {
+                const scope = scopes[i];
+                const language = grammar.embeddedLanguages[scope];
+                if (typeof language !== 'string') {
+                    continue;
+                }
+                if (this._languageService.isRegisteredLanguageId(language)) {
+                    embeddedLanguages[scope] = this._languageService.languageIdCodec.encodeLanguageId(language);
+                }
+            }
+        }
+        const tokenTypes = Object.create(null);
+        if (grammar.tokenTypes) {
+            const scopes = ( Object.keys(grammar.tokenTypes));
+            for (const scope of scopes) {
+                const tokenType = grammar.tokenTypes[scope];
+                switch (tokenType) {
+                    case 'string':
+                        tokenTypes[scope] = 2 ;
+                        break;
+                    case 'other':
+                        tokenTypes[scope] = 0 ;
+                        break;
+                    case 'comment':
+                        tokenTypes[scope] = 1 ;
+                        break;
+                }
+            }
+        }
+        let validLanguageId = null;
+        if (grammar.language && this._languageService.isRegisteredLanguageId(grammar.language)) {
+            validLanguageId = grammar.language;
+        }
+        function asStringArray(array, defaultValue) {
+            if (!Array.isArray(array)) {
+                return defaultValue;
+            }
+            if (!array.every(e => typeof e === 'string')) {
+                return defaultValue;
+            }
+            return array;
+        }
+        return {
+            location: grammarLocation,
+            language: validLanguageId || undefined,
+            scopeName: grammar.scopeName,
+            embeddedLanguages: embeddedLanguages,
+            tokenTypes: tokenTypes,
+            injectTo: grammar.injectTo,
+            balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
+            unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
+        };
+    }
+    startDebugMode(printFn, onStop) {
+        if (this._debugMode) {
+            this._notificationService.error(( nls.localize('alreadyDebugging', "Already Logging.")));
+            return;
+        }
+        this._debugModePrintFunc = printFn;
+        this._debugMode = true;
+        if (this._debugMode) {
+            this._progressService.withProgress({
+                location: 15 ,
+                buttons: [( nls.localize('stop', "Stop"))]
+            }, (progress) => {
+                progress.report({
+                    message: ( nls.localize(
+                        'progress1',
+                        "Preparing to log TM Grammar parsing. Press Stop when finished."
+                    ))
+                });
+                return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
+                    vscodeOniguruma.setDefaultDebugCall(true);
+                    progress.report({
+                        message: ( nls.localize('progress2', "Now logging TM Grammar parsing. Press Stop when finished."))
+                    });
+                    return ( new Promise((resolve, reject) => { }));
+                });
+            }, (choice) => {
+                this._getVSCodeOniguruma().then((vscodeOniguruma) => {
+                    this._debugModePrintFunc = () => { };
+                    this._debugMode = false;
+                    vscodeOniguruma.setDefaultDebugCall(false);
+                    onStop();
+                });
+            });
+        }
+    }
+    _canCreateGrammarFactory() {
+        return !!this._grammarDefinitions;
+    }
+    async _getOrCreateGrammarFactory() {
+        if (this._grammarFactory) {
+            return this._grammarFactory;
+        }
+        const [vscodeTextmate, vscodeOniguruma] = await Promise.all([import('vscode-textmate').then(module => module.default ?? module), this._getVSCodeOniguruma()]);
+        const onigLib = Promise.resolve({
+            createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
+            createOnigString: (str) => vscodeOniguruma.createOnigString(str)
+        });
+        if (this._grammarFactory) {
+            return this._grammarFactory;
+        }
+        this._grammarFactory = ( new TMGrammarFactory({
+            logTrace: (msg) => this._logService.trace(msg),
+            logError: (msg, err) => this._logService.error(msg, err),
+            readFile: (resource) => this._extensionResourceLoaderService.readExtensionResource(resource)
+        }, this._grammarDefinitions || [], vscodeTextmate, onigLib));
+        this._updateTheme(this._themeService.getColorTheme(), true);
+        return this._grammarFactory;
+    }
+    async createTokenizationSupport(languageId) {
+        if (!this._languageService.isRegisteredLanguageId(languageId)) {
+            return null;
+        }
+        if (!this._canCreateGrammarFactory()) {
+            return null;
+        }
+        try {
+            const grammarFactory = await this._getOrCreateGrammarFactory();
+            if (!grammarFactory.has(languageId)) {
+                return null;
+            }
+            const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
+            const r = await grammarFactory.createGrammar(languageId, encodedLanguageId);
+            if (!r.grammar) {
+                return null;
+            }
+            const maxTokenizationLineLength = observableConfigValue('editor.maxTokenizationLineLength', languageId, -1, this._configurationService);
+            const tokenization = ( new TextMateTokenizationSupport(
+                r.grammar,
+                r.initialState,
+                r.containsEmbeddedLanguages,
+                (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
+                () => this._configurationService.getValue('editor.experimental.asyncTokenizationVerification')
+            ));
+            tokenization.onDidEncounterLanguage((encodedLanguageId) => {
+                if (!this._encounteredLanguages[encodedLanguageId]) {
+                    const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
+                    this._encounteredLanguages[encodedLanguageId] = true;
+                    this._languageService.requestBasicLanguageFeatures(languageId);
+                }
+            });
+            return ( new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, maxTokenizationLineLength));
+        }
+        catch (err) {
+            if (err.message && err.message === missingTMGrammarErrorMessage) {
+                return null;
+            }
+            onUnexpectedError(err);
+            return null;
+        }
+    }
+    _updateTheme(colorTheme, forceUpdate) {
+        if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
+            && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
+            return;
+        }
+        this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
+        this._currentTokenColorMap = colorTheme.tokenColorMap;
+        this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
+        const colorMap = toColorMap(this._currentTokenColorMap);
+        const cssRules = generateTokensCSSForColorMap(colorMap);
+        this._styleElement.textContent = cssRules;
+        TokenizationRegistry.setColorMap(colorMap);
+        if (this._currentTheme && this._currentTokenColorMap) {
+            this._workerHost.acceptTheme(this._currentTheme, this._currentTokenColorMap);
+        }
+    }
+    async createGrammar(languageId) {
+        if (!this._languageService.isRegisteredLanguageId(languageId)) {
+            return null;
+        }
+        const grammarFactory = await this._getOrCreateGrammarFactory();
+        if (!grammarFactory.has(languageId)) {
+            return null;
+        }
+        const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
+        const { grammar } = await grammarFactory.createGrammar(languageId, encodedLanguageId);
+        return grammar;
+    }
+    _getVSCodeOniguruma() {
+        if (!this._vscodeOniguruma) {
+            this._vscodeOniguruma = (async () => {
+                const [vscodeOniguruma, wasm] = await Promise.all([import('vscode-oniguruma').then(module => module.default ?? module), this._loadVSCodeOnigurumaWASM()]);
+                await vscodeOniguruma.loadWASM({
+                    data: wasm,
+                    print: (str) => {
+                        this._debugModePrintFunc(str);
+                    }
+                });
+                return vscodeOniguruma;
+            })();
+        }
+        return this._vscodeOniguruma;
+    }
+    async _loadVSCodeOnigurumaWASM() {
+        if (isWeb) {
+            const response = await fetch(( ( FileAccess.asBrowserUri('vscode-oniguruma/../onig.wasm')).toString(true)));
+            return await response.arrayBuffer();
+        }
+        else {
+            const response = await fetch(this._environmentService.isBuilt
+                ? ( ( FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`)).toString(true))
+                : ( ( FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`)).toString(true)));
+            return response;
+        }
+    }
+};
+TextMateTokenizationFeature = ( __decorate([
+    ( __param(0, ILanguageService)),
+    ( __param(1, IWorkbenchThemeService)),
+    ( __param(2, IExtensionResourceLoaderService)),
+    ( __param(3, INotificationService)),
+    ( __param(4, ILogService)),
+    ( __param(5, IConfigurationService)),
+    ( __param(6, IProgressService)),
+    ( __param(7, IWorkbenchEnvironmentService)),
+    ( __param(8, IInstantiationService))
+], TextMateTokenizationFeature));
+function toColorMap(colorMap) {
+    const result = [null];
+    for (let i = 1, len = colorMap.length; i < len; i++) {
+        result[i] = ( Color.fromHex(colorMap[i]));
+    }
+    return result;
+}
+function equalsTokenRules(a, b) {
+    if (!b || !a || b.length !== a.length) {
+        return false;
+    }
+    for (let i = b.length - 1; i >= 0; i--) {
+        const r1 = b[i];
+        const r2 = a[i];
+        if (r1.scope !== r2.scope) {
+            return false;
+        }
+        const s1 = r1.settings;
+        const s2 = r2.settings;
+        if (s1 && s2) {
+            if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
+                return false;
+            }
+        }
+        else if (!s1 || !s2) {
+            return false;
+        }
+    }
+    return true;
+}
+function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
+    if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
+        collector.error(( nls.localize(
+            'invalid.language',
+            "Unknown language in `contributes.{0}.language`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.language)
+        )));
+        return false;
+    }
+    if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
+        collector.error(( nls.localize(
+            'invalid.scopeName',
+            "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.scopeName)
+        )));
+        return false;
+    }
+    if (!syntax.path || (typeof syntax.path !== 'string')) {
+        collector.error(( nls.localize(
+            'invalid.path.0',
+            "Expected string in `contributes.{0}.path`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.path)
+        )));
+        return false;
+    }
+    if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
+        collector.error(( nls.localize(
+            'invalid.injectTo',
+            "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.injectTo)
+        )));
+        return false;
+    }
+    if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
+        collector.error(( nls.localize(
+            'invalid.embeddedLanguages',
+            "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.embeddedLanguages)
+        )));
+        return false;
+    }
+    if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
+        collector.error(( nls.localize(
+            'invalid.tokenTypes',
+            "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.tokenTypes)
+        )));
+        return false;
+    }
+    const grammarLocation = resources.joinPath(extensionLocation, syntax.path);
+    if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) {
+        collector.warn(( nls.localize(
+            'invalid.path.1',
+            "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
+            grammarsExtPoint.name,
+            grammarLocation.path,
+            extensionLocation.path
+        )));
+    }
+    return true;
+}
+function observableConfigValue(key, languageId, defaultValue, configurationService) {
+    return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
+        if (e.affectsConfiguration(key, { overrideIdentifier: languageId })) {
+            handleChange(e);
+        }
+    }), () => configurationService.getValue(key, { overrideIdentifier: languageId }) ?? defaultValue);
+}
+export { TextMateTokenizationFeature };

package/dist/vscode/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js

@@ -0,0 +1,60 @@
+import { Emitter } from 'monaco-editor/esm/vs/base/common/event.js';
+import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
+import { TokenMetadata } from 'monaco-editor/esm/vs/editor/common/encodedTokenAttributes.js';
+import { EncodedTokenizationResult } from 'monaco-editor/esm/vs/editor/common/languages.js';
+class TextMateTokenizationSupport extends Disposable {
+    constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer, _backgroundTokenizerShouldOnlyVerifyTokens = () => false) {
+        super();
+        this._grammar = _grammar;
+        this._initialState = _initialState;
+        this._containsEmbeddedLanguages = _containsEmbeddedLanguages;
+        this._createBackgroundTokenizer = _createBackgroundTokenizer;
+        this._backgroundTokenizerShouldOnlyVerifyTokens = _backgroundTokenizerShouldOnlyVerifyTokens;
+        this._seenLanguages = [];
+        this._onDidEncounterLanguage = this._register(( new Emitter()));
+        this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
+    }
+    get backgroundTokenizerShouldOnlyVerifyTokens() {
+        return this._backgroundTokenizerShouldOnlyVerifyTokens();
+    }
+    getInitialState() {
+        return this._initialState;
+    }
+    tokenize(line, hasEOL, state) {
+        throw new Error('Not supported!');
+    }
+    createBackgroundTokenizer(textModel, store) {
+        if (this._createBackgroundTokenizer) {
+            return this._createBackgroundTokenizer(textModel, store);
+        }
+        return undefined;
+    }
+    tokenizeEncoded(line, hasEOL, state) {
+        const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
+        if (textMateResult.stoppedEarly) {
+            console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
+            return ( new EncodedTokenizationResult(textMateResult.tokens, state));
+        }
+        if (this._containsEmbeddedLanguages) {
+            const seenLanguages = this._seenLanguages;
+            const tokens = textMateResult.tokens;
+            for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
+                const metadata = tokens[(i << 1) + 1];
+                const languageId = TokenMetadata.getLanguageId(metadata);
+                if (!seenLanguages[languageId]) {
+                    seenLanguages[languageId] = true;
+                    this._onDidEncounterLanguage.fire(languageId);
+                }
+            }
+        }
+        let endState;
+        if (state.equals(textMateResult.ruleStack)) {
+            endState = state;
+        }
+        else {
+            endState = textMateResult.ruleStack;
+        }
+        return ( new EncodedTokenizationResult(textMateResult.tokens, endState));
+    }
+}
+export { TextMateTokenizationSupport };

package/dist/vscode/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js

@@ -0,0 +1,36 @@
+import { nullTokenizeEncoded } from 'monaco-editor/esm/vs/editor/common/languages/nullTokenize.js';
+import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
+import { keepAlive } from 'monaco-editor/esm/vs/base/common/observable.js';
+class TokenizationSupportWithLineLimit extends Disposable {
+    get backgroundTokenizerShouldOnlyVerifyTokens() {
+        return this._actual.backgroundTokenizerShouldOnlyVerifyTokens;
+    }
+    constructor(_encodedLanguageId, _actual, _maxTokenizationLineLength) {
+        super();
+        this._encodedLanguageId = _encodedLanguageId;
+        this._actual = _actual;
+        this._maxTokenizationLineLength = _maxTokenizationLineLength;
+        this._register(keepAlive(this._maxTokenizationLineLength));
+    }
+    getInitialState() {
+        return this._actual.getInitialState();
+    }
+    tokenize(line, hasEOL, state) {
+        throw new Error('Not supported!');
+    }
+    tokenizeEncoded(line, hasEOL, state) {
+        if (line.length >= this._maxTokenizationLineLength.get()) {
+            return nullTokenizeEncoded(this._encodedLanguageId, state);
+        }
+        return this._actual.tokenizeEncoded(line, hasEOL, state);
+    }
+    createBackgroundTokenizer(textModel, store) {
+        if (this._actual.createBackgroundTokenizer) {
+            return this._actual.createBackgroundTokenizer(textModel, store);
+        }
+        else {
+            return undefined;
+        }
+    }
+}
+export { TokenizationSupportWithLineLimit };