@codingame/monaco-vscode-api 1.76.9 → 1.78.0-next.1

package/dist/textmate.js

@@ -1,53 +1,57 @@
-import { o as onServicesInitialized, u as nodeModulesAsarUnpackedPath, v as nodeModulesPath, x as registerAssets, y as nodeModulesAsarPath } from './polyfill.js';
-import { bW as ITextMateTokenizationService } from './missing-services.js';
-import { E as Emitter, _ as __decorate, a as __param, o as IWorkbenchEnvironmentService } from './services2.js';
-import { I as IExtensionResourceLoaderService } from './extensionResourceLoader.js';
-import { Disposable, DisposableStore, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
-import { TokenizationRegistry, EncodedTokenizationResult } from 'monaco-editor/esm/vs/editor/common/languages.js';
+import { __decorate, Extensions$2, __param, ITextMateTokenizationService, IExtensionResourceLoaderService, nodeModulesAsarUnpackedPath, nodeModulesPath, IWorkbenchEnvironmentService, registerServiceInitializeParticipant, Emitter, LineRange, Range$2, nodeModulesAsarPath, registerAssets } from './polyfill.js';
+import { DisposableStore, Disposable, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
+import { TokenizationRegistry, LazyTokenizationSupport, EncodedTokenizationResult } from 'monaco-editor/esm/vs/editor/common/languages.js';
 import { TokenMetadata } from 'monaco-editor/esm/vs/editor/common/encodedTokenAttributes.js';
+import { observableFromEvent, keepAlive, autorun } from './utils.js';
 import { nullTokenizeEncoded } from 'monaco-editor/esm/vs/editor/common/languages/nullTokenize.js';
-import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
 import { equals, compareBy, numberComparator } from 'monaco-editor/esm/vs/base/common/arrays.js';
 import { INITIAL, applyStateStackDiff } from 'vscode-textmate';
-import { ContiguousGrowingArray } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
+import { TokenizationStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
 import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
 import { countEOL } from 'monaco-editor/esm/vs/editor/common/core/eolCounter.js';
 import { createWebWorker } from 'monaco-editor/esm/vs/editor/browser/services/webWorker.js';
-import { onUnexpectedError, BugIndicatingError } from 'monaco-editor/esm/vs/base/common/errors.js';
+import { BugIndicatingError, onUnexpectedError } from 'monaco-editor/esm/vs/base/common/errors.js';
 import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
 import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
 import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
+import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
 import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
 import { IEnvironmentService } from 'monaco-editor/esm/vs/platform/environment/common/environment.js';
+import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
+import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
 import { FileAccess } from 'monaco-editor/esm/vs/base/common/network.js';
 import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
 import { joinPath, isEqualOrParent, isEqual } from 'monaco-editor/esm/vs/base/common/resources.js';
-import { E as ExtensionsRegistry } from './extensionsRegistry.js';
-import { l as languagesExtPoint } from './languageService.js';
+import { ExtensionsRegistry } from './extensionsRegistry.js';
+import { languagesExtPoint } from './languageService.js';
 import { localize } from 'monaco-editor/esm/vs/nls.js';
-import { I as IWorkbenchThemeService } from './workbenchThemeService.js';
+import { IWorkbenchThemeService, getTokenClassificationRegistry, typeAndModifierIdPattern } from './tokenClassificationRegistry.js';
 import { createStyleSheet } from 'monaco-editor/esm/vs/base/browser/dom.js';
 import { generateTokensCSSForColorMap } from 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js';
 import { isObject } from 'monaco-editor/esm/vs/base/common/types.js';
-import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
 import { ILogService } from 'monaco-editor/esm/vs/platform/log/common/log.js';
 import { IProgressService } from 'monaco-editor/esm/vs/platform/progress/common/progress.js';
 import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
 import { Color } from 'monaco-editor/esm/vs/base/common/color.js';
 import _onigWasm from 'vscode-oniguruma/release/onig.wasm';
-import { g as getServiceOverride$1 } from './files.js';
+import getServiceOverride$1 from './files.js';
+import { Registry } from 'monaco-editor/esm/vs/platform/registry/common/platform.js';
 import { SyncDescriptor } from 'monaco-editor/esm/vs/platform/instantiation/common/descriptors.js';
 class TextMateTokenizationSupport extends Disposable {
-    constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer) {
+    constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer, _backgroundTokenizerShouldOnlyVerifyTokens = () => false) {
         super();
         this._grammar = _grammar;
         this._initialState = _initialState;
         this._containsEmbeddedLanguages = _containsEmbeddedLanguages;
         this._createBackgroundTokenizer = _createBackgroundTokenizer;
+        this._backgroundTokenizerShouldOnlyVerifyTokens = _backgroundTokenizerShouldOnlyVerifyTokens;
         this._seenLanguages = [];
         this._onDidEncounterLanguage = this._register(( (new Emitter())));
         this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
     }
+    get backgroundTokenizerShouldOnlyVerifyTokens() {
+        return this._backgroundTokenizerShouldOnlyVerifyTokens();
+    }
     getInitialState() {
         return this._initialState;
     }
@@ -92,22 +96,16 @@ class TextMateTokenizationSupport extends Disposable {
         );
     }
 }
-let TokenizationSupportWithLineLimit = class TokenizationSupportWithLineLimit {
-    constructor(_languageId, _encodedLanguageId, _actual, _configurationService) {
-        this._languageId = _languageId;
+class TokenizationSupportWithLineLimit extends Disposable {
+    get backgroundTokenizerShouldOnlyVerifyTokens() {
+        return this._actual.backgroundTokenizerShouldOnlyVerifyTokens;
+    }
+    constructor(_encodedLanguageId, _actual, _maxTokenizationLineLength) {
+        super();
         this._encodedLanguageId = _encodedLanguageId;
         this._actual = _actual;
-        this._configurationService = _configurationService;
-        this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
-            overrideIdentifier: this._languageId
-        });
-        this._configurationService.onDidChangeConfiguration(e => {
-            if (e.affectsConfiguration('editor.maxTokenizationLineLength')) {
-                this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
-                    overrideIdentifier: this._languageId
-                });
-            }
-        });
+        this._maxTokenizationLineLength = _maxTokenizationLineLength;
+        this._register(keepAlive(this._maxTokenizationLineLength));
     }
     getInitialState() {
         return this._actual.getInitialState();
@@ -116,7 +114,7 @@ let TokenizationSupportWithLineLimit = class TokenizationSupportWithLineLimit {
         throw new Error('Not supported!');
     }
     tokenizeEncoded(line, hasEOL, state) {
-        if (line.length >= this._maxTokenizationLineLength) {
+        if (line.length >= this._maxTokenizationLineLength.get()) {
             return nullTokenizeEncoded(this._encodedLanguageId, state);
         }
         return this._actual.tokenizeEncoded(line, hasEOL, state);
@@ -129,22 +127,12 @@ let TokenizationSupportWithLineLimit = class TokenizationSupportWithLineLimit {
             return undefined;
         }
     }
-};
-TokenizationSupportWithLineLimit = ( (__decorate([
-    ( (__param(3, IConfigurationService)))
-], TokenizationSupportWithLineLimit)));
+}
 class ArrayEdit {
     constructor(
     edits) {
         this.edits = edits.slice().sort(compareBy(c => c.offset, numberComparator));
     }
-    applyTo(array) {
-        for (let i = this.edits.length - 1; i >= 0; i--) {
-            const c = this.edits[i];
-            array.delete(c.offset, c.length);
-            array.insert(c.offset, c.newLength);
-        }
-    }
     applyToArray(array) {
         for (let i = this.edits.length - 1; i >= 0; i--) {
             const c = this.edits[i];
@@ -203,16 +191,26 @@ class CombinedIndexTransformer {
     }
 }
 class TextMateWorkerTokenizerController extends Disposable {
-    constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _initialState) {
+    constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _initialState, _configurationService, _maxTokenizationLineLength) {
         super();
         this._model = _model;
         this._worker = _worker;
         this._languageIdCodec = _languageIdCodec;
         this._backgroundTokenizationStore = _backgroundTokenizationStore;
         this._initialState = _initialState;
+        this._configurationService = _configurationService;
+        this._maxTokenizationLineLength = _maxTokenizationLineLength;
         this._pendingChanges = [];
-        this._states = ( (new ContiguousGrowingArray(null)));
+        this._states = ( (new TokenizationStateStore()));
+        this._loggingEnabled = observableConfigValue$1('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
+        this._register(keepAlive(this._loggingEnabled));
         this._register(this._model.onDidChangeContent((e) => {
+            if (this.shouldLog) {
+                console.log('model change', {
+                    fileName: this._model.uri.fsPath.split('\\').pop(),
+                    changes: changesToString(e.changes),
+                });
+            }
             this._worker.acceptModelChanged(( (this._model.uri.toString())), e);
             this._pendingChanges.push(e);
         }));
@@ -230,21 +228,45 @@ class TextMateWorkerTokenizerController extends Disposable {
             EOL: this._model.getEOL(),
             languageId,
             encodedLanguageId,
+            maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
         });
+        this._register(autorun('update maxTokenizationLineLength', reader => {
+            const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
+            this._worker.acceptMaxTokenizationLineLength(( (this._model.uri.toString())), maxTokenizationLineLength);
+        }));
+    }
+    get shouldLog() {
+        return this._loggingEnabled.get();
     }
     dispose() {
         super.dispose();
         this._worker.acceptRemovedModel(( (this._model.uri.toString())));
     }
     setTokensAndStates(versionId, rawTokens, stateDeltas) {
+        let tokens = ContiguousMultilineTokensBuilder.deserialize(( (new Uint8Array(rawTokens))));
+        if (this.shouldLog) {
+            console.log('received background tokenization result', {
+                fileName: this._model.uri.fsPath.split('\\').pop(),
+                updatedTokenLines: ( (tokens.map((t) => t.getLineRange()))).join(' & '),
+                updatedStateLines: ( (stateDeltas.map(
+                    (s) => ( (( (new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length))).toString()))
+                ))).join(' & '),
+            });
+        }
+        if (this.shouldLog) {
+            const changes = ( (( (this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes))).map(c => changesToString(c)))).join(' then ');
+            console.log('Applying changes to local states', changes);
+        }
         while (this._pendingChanges.length > 0 &&
             this._pendingChanges[0].versionId <= versionId) {
             const change = this._pendingChanges.shift();
-            const op = lineArrayEditFromModelContentChange(change.changes);
-            op.applyTo(this._states);
+            this._states.acceptChanges(change.changes);
         }
-        let tokens = ContiguousMultilineTokensBuilder.deserialize(( (new Uint8Array(rawTokens))));
         if (this._pendingChanges.length > 0) {
+            if (this.shouldLog) {
+                const changes = ( (( (this._pendingChanges.map(c => c.changes))).map(c => changesToString(c)))).join(' then ');
+                console.log('Considering non-processed changes', changes);
+            }
             const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( (this._pendingChanges.map(
                 (c) => ( (new ArrayEdit( (c.changes.map((c) => ( (new SingleArrayEdit(
                     c.range.startLineNumber - 1,
@@ -273,11 +295,17 @@ class TextMateWorkerTokenizerController extends Disposable {
         this._backgroundTokenizationStore.setTokens(tokens);
         const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( (this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes)))));
         for (const d of stateDeltas) {
-            let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.get(d.startLineNumber - 1 - 1);
+            let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.getEndState(d.startLineNumber - 1);
             for (let i = 0; i < d.stateDeltas.length; i++) {
                 const delta = d.stateDeltas[i];
-                const state = applyStateStackDiff(prevState, delta);
-                this._states.set(d.startLineNumber + i - 1, state);
+                let state;
+                if (delta) {
+                    state = applyStateStackDiff(prevState, delta);
+                    this._states.setEndState(d.startLineNumber + i, state);
+                }
+                else {
+                    state = this._states.getEndState(d.startLineNumber + i);
+                }
                 const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
                 if (offset !== undefined) {
                     this._backgroundTokenizationStore.setEndState(offset + 1, state);
@@ -299,14 +327,26 @@ function lineArrayEditFromModelContentChange(c) {
         )))))))
     );
 }
+function changesToString(changes) {
+    return ( (changes.map(c => ( (Range$2.lift(c.range).toString())) + ' => ' + c.text))).join(' & ');
+}
+function observableConfigValue$1(key, defaultValue, configurationService) {
+    return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
+        if (e.affectsConfiguration(key)) {
+            handleChange(e);
+        }
+    }), () => configurationService.getValue(key) ?? defaultValue);
+}
 let TextMateWorkerHost = class TextMateWorkerHost {
-    constructor(_extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService) {
+    constructor(_extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
         this._extensionResourceLoaderService = _extensionResourceLoaderService;
         this._modelService = _modelService;
         this._languageConfigurationService = _languageConfigurationService;
         this._configurationService = _configurationService;
         this._languageService = _languageService;
         this._environmentService = _environmentService;
+        this._notificationService = _notificationService;
+        this._telemetryService = _telemetryService;
         this._workerProxyPromise = null;
         this._worker = null;
         this._workerProxy = null;
@@ -382,7 +422,7 @@ let TextMateWorkerHost = class TextMateWorkerHost {
         this._workerProxy = null;
         this._workerProxyPromise = null;
     }
-    createBackgroundTokenizer(textModel, tokenStore) {
+    createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength) {
         if (this._workerTokenizerControllers.has(( (textModel.uri.toString())))) {
             throw new BugIndicatingError();
         }
@@ -404,7 +444,9 @@ let TextMateWorkerHost = class TextMateWorkerHost {
                     workerProxy,
                     this._languageService.languageIdCodec,
                     tokenStore,
-                    INITIAL
+                    INITIAL,
+                    this._configurationService,
+                    maxTokenizationLineLength
                 )));
                 this._workerTokenizerControllers.set(( (textModel.uri.toString())), controller);
                 return toDisposable(() => {
@@ -422,6 +464,17 @@ let TextMateWorkerHost = class TextMateWorkerHost {
                     workerProxy?.retokenize(( (textModel.uri.toString())), startLineNumber, endLineNumberExclusive);
                 });
             },
+            reportMismatchingTokens: (lineNumber) => {
+                if (TextMateWorkerHost._reportedMismatchingTokens) {
+                    return;
+                }
+                TextMateWorkerHost._reportedMismatchingTokens = true;
+                this._notificationService.error({
+                    message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
+                    name: 'Async Tokenization Token Mismatch',
+                });
+                this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
+            },
         };
     }
     async readFile(_resource) {
@@ -436,13 +489,16 @@ let TextMateWorkerHost = class TextMateWorkerHost {
         }
     }
 };
+TextMateWorkerHost._reportedMismatchingTokens = false;
 TextMateWorkerHost = ( (__decorate([
     ( (__param(0, IExtensionResourceLoaderService))),
     ( (__param(1, IModelService))),
     ( (__param(2, ILanguageConfigurationService))),
     ( (__param(3, IConfigurationService))),
     ( (__param(4, ILanguageService))),
-    ( (__param(5, IEnvironmentService)))
+    ( (__param(5, IEnvironmentService))),
+    ( (__param(6, INotificationService))),
+    ( (__param(7, ITelemetryService)))
 ], TextMateWorkerHost)));
 function keepAliveWhenAttached(textModel, factory) {
     const disposableStore = ( (new DisposableStore()));
@@ -698,8 +754,6 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
         this._progressService = _progressService;
         this._environmentService = _environmentService;
         this._instantiationService = _instantiationService;
-        this._onDidEncounterLanguage = this._register(( (new Emitter())));
-        this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
         this._createdModes = [];
         this._encounteredLanguages = [];
         this._debugMode = false;
@@ -718,7 +772,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
         this._register(this._themeService.onDidColorThemeChange(() => {
             this._updateTheme(this._themeService.getColorTheme(), false);
         }));
-        this._languageService.onDidEncounterLanguage((languageId) => {
+        this._languageService.onDidRequestRichLanguageFeatures((languageId) => {
             this._createdModes.push(languageId);
         });
     }
@@ -737,9 +791,9 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
                 if (def) {
                     this._grammarDefinitions.push(def);
                     if (def.language) {
-                        this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(def.language, {
-                            createTokenizationSupport: async () => this.createTokenizationSupport(def.language)
-                        }));
+                        const lazyTokenizationSupport = ( (new LazyTokenizationSupport(() => this.createTokenizationSupport(def.language))));
+                        this._tokenizersRegistrations.add(lazyTokenizationSupport);
+                        this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(def.language, lazyTokenizationSupport));
                     }
                 }
             }
@@ -887,21 +941,23 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
             if (!r.grammar) {
                 return null;
             }
+            const maxTokenizationLineLength = observableConfigValue('editor.maxTokenizationLineLength', languageId, -1, this._configurationService);
             const tokenization = ( (new TextMateTokenizationSupport(
                 r.grammar,
                 r.initialState,
                 r.containsEmbeddedLanguages,
-                (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore)
+                (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
+                () => this._configurationService.getValue('editor.experimental.asyncTokenizationVerification')
             )));
             tokenization.onDidEncounterLanguage((encodedLanguageId) => {
                 if (!this._encounteredLanguages[encodedLanguageId]) {
                     const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
                     this._encounteredLanguages[encodedLanguageId] = true;
-                    this._onDidEncounterLanguage.fire(languageId);
+                    this._languageService.requestBasicLanguageFeatures(languageId);
                 }
             });
             return (
-                 (new TokenizationSupportWithLineLimit(languageId, encodedLanguageId, tokenization, this._configurationService))
+                 (new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, maxTokenizationLineLength))
             );
         }
         catch (err) {
@@ -1076,14 +1132,294 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
     }
     return true;
 }
+function observableConfigValue(key, languageId, defaultValue, configurationService) {
+    return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
+        if (e.affectsConfiguration(key, { overrideIdentifier: languageId })) {
+            handleChange(e);
+        }
+    }), () => configurationService.getValue(key, { overrideIdentifier: languageId }) ?? defaultValue);
+}
+const tokenClassificationRegistry = getTokenClassificationRegistry();
+const tokenTypeExtPoint = ( (ExtensionsRegistry.registerExtensionPoint({
+    extensionPoint: 'semanticTokenTypes',
+    jsonSchema: {
+        description: ( (localize('contributes.semanticTokenTypes', 'Contributes semantic token types.'))),
+        type: 'array',
+        items: {
+            type: 'object',
+            properties: {
+                id: {
+                    type: 'string',
+                    description: ( (localize(
+                        'contributes.semanticTokenTypes.id',
+                        'The identifier of the semantic token type'
+                    ))),
+                    pattern: typeAndModifierIdPattern,
+                    patternErrorMessage: ( (localize(
+                        'contributes.semanticTokenTypes.id.format',
+                        'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'
+                    ))),
+                },
+                superType: {
+                    type: 'string',
+                    description: ( (localize(
+                        'contributes.semanticTokenTypes.superType',
+                        'The super type of the semantic token type'
+                    ))),
+                    pattern: typeAndModifierIdPattern,
+                    patternErrorMessage: ( (localize(
+                        'contributes.semanticTokenTypes.superType.format',
+                        'Super types should be in the form letterOrDigit[_-letterOrDigit]*'
+                    ))),
+                },
+                description: {
+                    type: 'string',
+                    description: ( (localize(
+                        'contributes.color.description',
+                        'The description of the semantic token type'
+                    ))),
+                }
+            }
+        }
+    }
+})));
+const tokenModifierExtPoint = ( (ExtensionsRegistry.registerExtensionPoint({
+    extensionPoint: 'semanticTokenModifiers',
+    jsonSchema: {
+        description: ( (localize(
+            'contributes.semanticTokenModifiers',
+            'Contributes semantic token modifiers.'
+        ))),
+        type: 'array',
+        items: {
+            type: 'object',
+            properties: {
+                id: {
+                    type: 'string',
+                    description: ( (localize(
+                        'contributes.semanticTokenModifiers.id',
+                        'The identifier of the semantic token modifier'
+                    ))),
+                    pattern: typeAndModifierIdPattern,
+                    patternErrorMessage: ( (localize(
+                        'contributes.semanticTokenModifiers.id.format',
+                        'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'
+                    )))
+                },
+                description: {
+                    description: ( (localize(
+                        'contributes.semanticTokenModifiers.description',
+                        'The description of the semantic token modifier'
+                    )))
+                }
+            }
+        }
+    }
+})));
+const tokenStyleDefaultsExtPoint = ( (ExtensionsRegistry.registerExtensionPoint({
+    extensionPoint: 'semanticTokenScopes',
+    jsonSchema: {
+        description: ( (localize(
+            'contributes.semanticTokenScopes',
+            'Contributes semantic token scope maps.'
+        ))),
+        type: 'array',
+        items: {
+            type: 'object',
+            properties: {
+                language: {
+                    description: ( (localize(
+                        'contributes.semanticTokenScopes.languages',
+                        'Lists the languge for which the defaults are.'
+                    ))),
+                    type: 'string'
+                },
+                scopes: {
+                    description: ( (localize(
+                        'contributes.semanticTokenScopes.scopes',
+                        'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
+                    ))),
+                    type: 'object',
+                    additionalProperties: {
+                        type: 'array',
+                        items: {
+                            type: 'string'
+                        }
+                    }
+                }
+            }
+        }
+    }
+})));
+class TokenClassificationExtensionPoints {
+    constructor() {
+        function validateTypeOrModifier(contribution, extensionPoint, collector) {
+            if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
+                collector.error(( (localize(
+                    'invalid.id',
+                    "'configuration.{0}.id' must be defined and can not be empty",
+                    extensionPoint
+                ))));
+                return false;
+            }
+            if (!contribution.id.match(typeAndModifierIdPattern)) {
+                collector.error(( (localize(
+                    'invalid.id.format',
+                    "'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
+                    extensionPoint
+                ))));
+                return false;
+            }
+            const superType = contribution.superType;
+            if (superType && !superType.match(typeAndModifierIdPattern)) {
+                collector.error(( (localize(
+                    'invalid.superType.format',
+                    "'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
+                    extensionPoint
+                ))));
+                return false;
+            }
+            if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
+                collector.error(( (localize(
+                    'invalid.description',
+                    "'configuration.{0}.description' must be defined and can not be empty",
+                    extensionPoint
+                ))));
+                return false;
+            }
+            return true;
+        }
+        tokenTypeExtPoint.setHandler((extensions, delta) => {
+            for (const extension of delta.added) {
+                const extensionValue = extension.value;
+                const collector = extension.collector;
+                if (!extensionValue || !Array.isArray(extensionValue)) {
+                    collector.error(( (localize(
+                        'invalid.semanticTokenTypeConfiguration',
+                        "'configuration.semanticTokenType' must be an array"
+                    ))));
+                    return;
+                }
+                for (const contribution of extensionValue) {
+                    if (validateTypeOrModifier(contribution, 'semanticTokenType', collector)) {
+                        tokenClassificationRegistry.registerTokenType(contribution.id, contribution.description, contribution.superType);
+                    }
+                }
+            }
+            for (const extension of delta.removed) {
+                const extensionValue = extension.value;
+                for (const contribution of extensionValue) {
+                    tokenClassificationRegistry.deregisterTokenType(contribution.id);
+                }
+            }
+        });
+        tokenModifierExtPoint.setHandler((extensions, delta) => {
+            for (const extension of delta.added) {
+                const extensionValue = extension.value;
+                const collector = extension.collector;
+                if (!extensionValue || !Array.isArray(extensionValue)) {
+                    collector.error(( (localize(
+                        'invalid.semanticTokenModifierConfiguration',
+                        "'configuration.semanticTokenModifier' must be an array"
+                    ))));
+                    return;
+                }
+                for (const contribution of extensionValue) {
+                    if (validateTypeOrModifier(contribution, 'semanticTokenModifier', collector)) {
+                        tokenClassificationRegistry.registerTokenModifier(contribution.id, contribution.description);
+                    }
+                }
+            }
+            for (const extension of delta.removed) {
+                const extensionValue = extension.value;
+                for (const contribution of extensionValue) {
+                    tokenClassificationRegistry.deregisterTokenModifier(contribution.id);
+                }
+            }
+        });
+        tokenStyleDefaultsExtPoint.setHandler((extensions, delta) => {
+            for (const extension of delta.added) {
+                const extensionValue = extension.value;
+                const collector = extension.collector;
+                if (!extensionValue || !Array.isArray(extensionValue)) {
+                    collector.error(( (localize(
+                        'invalid.semanticTokenScopes.configuration',
+                        "'configuration.semanticTokenScopes' must be an array"
+                    ))));
+                    return;
+                }
+                for (const contribution of extensionValue) {
+                    if (contribution.language && typeof contribution.language !== 'string') {
+                        collector.error(( (localize(
+                            'invalid.semanticTokenScopes.language',
+                            "'configuration.semanticTokenScopes.language' must be a string"
+                        ))));
+                        continue;
+                    }
+                    if (!contribution.scopes || typeof contribution.scopes !== 'object') {
+                        collector.error(( (localize(
+                            'invalid.semanticTokenScopes.scopes',
+                            "'configuration.semanticTokenScopes.scopes' must be defined as an object"
+                        ))));
+                        continue;
+                    }
+                    for (const selectorString in contribution.scopes) {
+                        const tmScopes = contribution.scopes[selectorString];
+                        if (!Array.isArray(tmScopes) || tmScopes.some(l => typeof l !== 'string')) {
+                            collector.error(( (localize(
+                                'invalid.semanticTokenScopes.scopes.value',
+                                "'configuration.semanticTokenScopes.scopes' values must be an array of strings"
+                            ))));
+                            continue;
+                        }
+                        try {
+                            const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
+                            tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( (tmScopes.map(s => s.split(' ')))) });
+                        }
+                        catch (e) {
+                            collector.error(( (localize(
+                                'invalid.semanticTokenScopes.scopes.selector',
+                                "configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
+                                selectorString
+                            ))));
+                        }
+                    }
+                }
+            }
+            for (const extension of delta.removed) {
+                const extensionValue = extension.value;
+                for (const contribution of extensionValue) {
+                    for (const selectorString in contribution.scopes) {
+                        const tmScopes = contribution.scopes[selectorString];
+                        try {
+                            const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
+                            tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( (tmScopes.map(s => s.split(' ')))) });
+                        }
+                        catch (e) {
+                        }
+                    }
+                }
+            }
+        });
+    }
+}
 registerAssets({
     'vscode-oniguruma/../onig.wasm': _onigWasm
 });
-function initialize(instantiationService) {
-    instantiationService.invokeFunction((accessor) => accessor.get(ITextMateTokenizationService));
-}
+registerServiceInitializeParticipant(async (accessor) => {
+    accessor.get(ITextMateTokenizationService);
+});
+let ExtensionPoints = class ExtensionPoints {
+    constructor(instantiationService) {
+        this.instantiationService = instantiationService;
+        this.instantiationService.createInstance(TokenClassificationExtensionPoints);
+    }
+};
+ExtensionPoints = __decorate([
+    ( __param(0, IInstantiationService))
+], ExtensionPoints);
+( Registry.as(Extensions$2.Workbench)).registerWorkbenchContribution(ExtensionPoints, 1 );
 function getServiceOverride() {
-    onServicesInitialized(initialize);
     return {
         ...getServiceOverride$1(),
         [( ITextMateTokenizationService.toString())]: new SyncDescriptor(TextMateTokenizationFeature)