@codingame/monaco-vscode-api 1.69.24 → 1.76.0-next.2

package/dist/textmate.js

@@ -1,124 +1,470 @@
-import { _ as __decorate, a as __param, E as Emitter } from './workspace.js';
+import { _ as __decorate, a as __param, E as Emitter } from './contextkey.js';
 import './polyfill.js';
-import { h as ITextMateService } from './missing-services.js';
+import { f as ITextMateTokenizationService } from './missing-services.js';
 import { g as getExtensionPoint, a as ExtensionMessageCollector, c as consoleExtensionMessageHandler, o as onServicesInitialized, E as ExtensionsRegistry } from './tools.js';
-import { l as languagesExtPoint } from './languageService.js';
-import { localize } from 'monaco-editor/esm/vs/nls.js';
-import { I as IWorkbenchThemeService } from './workbenchThemeService.js';
-import { Disposable, dispose } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
-import { joinPath, isEqualOrParent, isEqual } from 'monaco-editor/esm/vs/base/common/resources.js';
 import { r as registerExtensionFile, g as getServiceOverride$1, I as IExtensionResourceLoaderService } from './files.js';
+import { S as Services, l as IWorkbenchEnvironmentService } from './services2.js';
+import { Disposable, DisposableStore, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
 import { TokenizationRegistry, EncodedTokenizationResult } from 'monaco-editor/esm/vs/editor/common/languages.js';
 import { TokenMetadata } from 'monaco-editor/esm/vs/editor/common/encodedTokenAttributes.js';
+import { nullTokenizeEncoded } from 'monaco-editor/esm/vs/editor/common/languages/nullTokenize.js';
+import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
+import { equals, compareBy, numberComparator } from 'monaco-editor/esm/vs/base/common/arrays.js';
+import { INITIAL, applyStateStackDiff } from 'vscode-textmate';
+import { ContiguousGrowingArray } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
+import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
+import { countEOL } from 'monaco-editor/esm/vs/editor/common/core/eolCounter.js';
+import { createWebWorker } from 'monaco-editor/esm/vs/editor/browser/services/webWorker.js';
+import { onUnexpectedError, BugIndicatingError } from 'monaco-editor/esm/vs/base/common/errors.js';
+import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
+import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
+import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
+import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
+import { IEnvironmentService } from 'monaco-editor/esm/vs/platform/environment/common/environment.js';
+import { FileAccess } from 'monaco-editor/esm/vs/base/common/network.js';
+import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
+import { joinPath, isEqualOrParent, isEqual } from 'monaco-editor/esm/vs/base/common/resources.js';
+import { l as languagesExtPoint } from './languageService.js';
+import { localize } from 'monaco-editor/esm/vs/nls.js';
+import { I as IWorkbenchThemeService } from './workbenchThemeService.js';
 import { createStyleSheet } from 'monaco-editor/esm/vs/base/browser/dom.js';
-import { onUnexpectedError } from 'monaco-editor/esm/vs/base/common/errors.js';
-import { Color } from 'monaco-editor/esm/vs/base/common/color.js';
-import { equals } from 'monaco-editor/esm/vs/base/common/arrays.js';
 import { generateTokensCSSForColorMap } from 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js';
-import { isObject } from 'monaco-editor/esm/vs/base/common/types.js';
-import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
 import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
 import { ILogService } from 'monaco-editor/esm/vs/platform/log/common/log.js';
-import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
 import { IProgressService } from 'monaco-editor/esm/vs/platform/progress/common/progress.js';
-import { nullTokenizeEncoded } from 'monaco-editor/esm/vs/editor/common/languages/nullTokenize.js';
-import { S as Services } from './services2.js';
-import { be as DEFAULT_EXTENSION } from './extHost.js';
+import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
+import { isObject } from 'monaco-editor/esm/vs/base/common/types.js';
+import { Color } from 'monaco-editor/esm/vs/base/common/color.js';
 import { c as createInjectedClass } from './injection.js';
+import { bj as DEFAULT_EXTENSION } from './extHost.js';
 import { SyncDescriptor } from 'monaco-editor/esm/vs/platform/instantiation/common/descriptors.js';
-import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
-const grammarsExtPoint = ( (ExtensionsRegistry.registerExtensionPoint({
-    extensionPoint: 'grammars',
-    deps: [languagesExtPoint],
-    jsonSchema: {
-        description: ( (localize(
-            'vscode.extension.contributes.grammars',
-            'Contributes textmate tokenizers.'
-        ))),
-        type: 'array',
-        defaultSnippets: [{ body: [{ language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' }] }],
-        items: {
-            type: 'object',
-            defaultSnippets: [{ body: { language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' } }],
-            properties: {
-                language: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.language',
-                        'Language identifier for which this syntax is contributed to.'
-                    ))),
-                    type: 'string'
-                },
-                scopeName: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.scopeName',
-                        'Textmate scope name used by the tmLanguage file.'
-                    ))),
-                    type: 'string'
-                },
-                path: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.path',
-                        'Path of the tmLanguage file. The path is relative to the extension folder and typically starts with \'./syntaxes/\'.'
-                    ))),
-                    type: 'string'
-                },
-                embeddedLanguages: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.embeddedLanguages',
-                        'A map of scope name to language id if this grammar contains embedded languages.'
-                    ))),
-                    type: 'object'
-                },
-                tokenTypes: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.tokenTypes',
-                        'A map of scope name to token types.'
-                    ))),
-                    type: 'object',
-                    additionalProperties: {
-                        enum: ['string', 'comment', 'other']
+const nodeModulesPath = 'vs/../../node_modules';
+const nodeModulesAsarPath = 'vs/../../node_modules.asar';
+const nodeModulesAsarUnpackedPath = 'vs/../../node_modules.asar.unpacked';
+class TextMateTokenizationSupport extends Disposable {
+    constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer) {
+        super();
+        this._grammar = _grammar;
+        this._initialState = _initialState;
+        this._containsEmbeddedLanguages = _containsEmbeddedLanguages;
+        this._createBackgroundTokenizer = _createBackgroundTokenizer;
+        this._seenLanguages = [];
+        this._onDidEncounterLanguage = this._register(( (new Emitter())));
+        this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
+    }
+    getInitialState() {
+        return this._initialState;
+    }
+    tokenize(line, hasEOL, state) {
+        throw new Error('Not supported!');
+    }
+    createBackgroundTokenizer(textModel, store) {
+        if (this._createBackgroundTokenizer) {
+            return this._createBackgroundTokenizer(textModel, store);
+        }
+        return undefined;
+    }
+    tokenizeEncoded(line, hasEOL, state) {
+        const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
+        if (textMateResult.stoppedEarly) {
+            console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
+            return (
+                 (new EncodedTokenizationResult(textMateResult.tokens, state))
+            );
+        }
+        if (this._containsEmbeddedLanguages) {
+            const seenLanguages = this._seenLanguages;
+            const tokens = textMateResult.tokens;
+            for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
+                const metadata = tokens[(i << 1) + 1];
+                const languageId = TokenMetadata.getLanguageId(metadata);
+                if (!seenLanguages[languageId]) {
+                    seenLanguages[languageId] = true;
+                    this._onDidEncounterLanguage.fire(languageId);
+                }
+            }
+        }
+        let endState;
+        if (state.equals(textMateResult.ruleStack)) {
+            endState = state;
+        }
+        else {
+            endState = textMateResult.ruleStack;
+        }
+        return (
+             (new EncodedTokenizationResult(textMateResult.tokens, endState))
+        );
+    }
+}
+let TokenizationSupportWithLineLimit = class TokenizationSupportWithLineLimit {
+    constructor(_languageId, _encodedLanguageId, _actual, _configurationService) {
+        this._languageId = _languageId;
+        this._encodedLanguageId = _encodedLanguageId;
+        this._actual = _actual;
+        this._configurationService = _configurationService;
+        this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
+            overrideIdentifier: this._languageId
+        });
+        this._configurationService.onDidChangeConfiguration(e => {
+            if (e.affectsConfiguration('editor.maxTokenizationLineLength')) {
+                this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
+                    overrideIdentifier: this._languageId
+                });
+            }
+        });
+    }
+    getInitialState() {
+        return this._actual.getInitialState();
+    }
+    tokenize(line, hasEOL, state) {
+        throw new Error('Not supported!');
+    }
+    tokenizeEncoded(line, hasEOL, state) {
+        if (line.length >= this._maxTokenizationLineLength) {
+            return nullTokenizeEncoded(this._encodedLanguageId, state);
+        }
+        return this._actual.tokenizeEncoded(line, hasEOL, state);
+    }
+    createBackgroundTokenizer(textModel, store) {
+        if (this._actual.createBackgroundTokenizer) {
+            return this._actual.createBackgroundTokenizer(textModel, store);
+        }
+        else {
+            return undefined;
+        }
+    }
+};
+TokenizationSupportWithLineLimit = ( (__decorate([
+    ( (__param(3, IConfigurationService)))
+], TokenizationSupportWithLineLimit)));
+class ArrayEdit {
+    constructor(
+    edits) {
+        this.edits = edits.slice().sort(compareBy(c => c.offset, numberComparator));
+    }
+    applyTo(array) {
+        for (let i = this.edits.length - 1; i >= 0; i--) {
+            const c = this.edits[i];
+            array.delete(c.offset, c.length);
+            array.insert(c.offset, c.newLength);
+        }
+    }
+    applyToArray(array) {
+        for (let i = this.edits.length - 1; i >= 0; i--) {
+            const c = this.edits[i];
+            array.splice(c.offset, c.length, ...( (new Array(c.newLength))));
+        }
+    }
+}
+class SingleArrayEdit {
+    constructor(offset, length, newLength) {
+        this.offset = offset;
+        this.length = length;
+        this.newLength = newLength;
+    }
+    toString() {
+        return `[${this.offset}, +${this.length}) -> +${this.newLength}}`;
+    }
+}
+class MonotonousIndexTransformer {
+    static fromMany(transformations) {
+        const transformers = transformations.map(t => ( (new MonotonousIndexTransformer(t))));
+        return (
+             (new CombinedIndexTransformer(transformers))
+        );
+    }
+    constructor(transformation) {
+        this.transformation = transformation;
+        this.idx = 0;
+        this.offset = 0;
+    }
+    transform(index) {
+        let nextChange = this.transformation.edits[this.idx];
+        while (nextChange && nextChange.offset + nextChange.length <= index) {
+            this.offset += nextChange.newLength - nextChange.length;
+            this.idx++;
+            nextChange = this.transformation.edits[this.idx];
+        }
+        if (nextChange && nextChange.offset <= index) {
+            return undefined;
+        }
+        return index + this.offset;
+    }
+}
+class CombinedIndexTransformer {
+    constructor(transformers) {
+        this.transformers = transformers;
+    }
+    transform(index) {
+        for (const transformer of this.transformers) {
+            const result = transformer.transform(index);
+            if (result === undefined) {
+                return undefined;
+            }
+            index = result;
+        }
+        return index;
+    }
+}
+class TextMateWorkerTokenizerController extends Disposable {
+    constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _initialState) {
+        super();
+        this._model = _model;
+        this._worker = _worker;
+        this._languageIdCodec = _languageIdCodec;
+        this._backgroundTokenizationStore = _backgroundTokenizationStore;
+        this._initialState = _initialState;
+        this._pendingChanges = [];
+        this._states = ( (new ContiguousGrowingArray(null)));
+        this._register(this._model.onDidChangeContent((e) => {
+            this._worker.acceptModelChanged(( (this._model.uri.toString())), e);
+            this._pendingChanges.push(e);
+        }));
+        this._register(this._model.onDidChangeLanguage((e) => {
+            const languageId = this._model.getLanguageId();
+            const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
+            this._worker.acceptModelLanguageChanged(( (this._model.uri.toString())), languageId, encodedLanguageId);
+        }));
+        const languageId = this._model.getLanguageId();
+        const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
+        this._worker.acceptNewModel({
+            uri: this._model.uri,
+            versionId: this._model.getVersionId(),
+            lines: this._model.getLinesContent(),
+            EOL: this._model.getEOL(),
+            languageId,
+            encodedLanguageId,
+        });
+    }
+    dispose() {
+        super.dispose();
+        this._worker.acceptRemovedModel(( (this._model.uri.toString())));
+    }
+    setTokensAndStates(versionId, rawTokens, stateDeltas) {
+        while (this._pendingChanges.length > 0 &&
+            this._pendingChanges[0].versionId <= versionId) {
+            const change = this._pendingChanges.shift();
+            const op = lineArrayEditFromModelContentChange(change.changes);
+            op.applyTo(this._states);
+        }
+        let tokens = ContiguousMultilineTokensBuilder.deserialize(( (new Uint8Array(rawTokens))));
+        if (this._pendingChanges.length > 0) {
+            const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(this._pendingChanges.map((c) => ( (new ArrayEdit(c.changes.map((c) => ( (new SingleArrayEdit(
+                c.range.startLineNumber - 1,
+                (c.range.endLineNumber - c.range.startLineNumber) + 1,
+                countEOL(c.text)[0] + 1
+            )))))))));
+            const b = ( (new ContiguousMultilineTokensBuilder()));
+            for (const t of tokens) {
+                for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
+                    const result = curToFutureTransformerTokens.transform(i - 1);
+                    if (result !== undefined) {
+                        b.add(i, t.getLineTokens(i));
                     }
-                },
-                injectTo: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.injectTo',
-                        'List of language scope names to which this grammar is injected to.'
-                    ))),
-                    type: 'array',
-                    items: {
-                        type: 'string'
+                }
+            }
+            tokens = b.finalize();
+            for (const change of this._pendingChanges) {
+                for (const innerChanges of change.changes) {
+                    for (let j = 0; j < tokens.length; j++) {
+                        tokens[j].applyEdit(innerChanges.range, innerChanges.text);
                     }
-                },
-                balancedBracketScopes: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.balancedBracketScopes',
-                        'Defines which scope names contain balanced brackets.'
-                    ))),
-                    type: 'array',
-                    items: {
-                        type: 'string'
-                    },
-                    default: ['*'],
-                },
-                unbalancedBracketScopes: {
-                    description: ( (localize(
-                        'vscode.extension.contributes.grammars.unbalancedBracketScopes',
-                        'Defines which scope names do not contain balanced brackets.'
-                    ))),
-                    type: 'array',
-                    items: {
-                        type: 'string'
-                    },
-                    default: [],
-                },
+                }
+            }
+        }
+        this._backgroundTokenizationStore.setTokens(tokens);
+        const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes)));
+        for (const d of stateDeltas) {
+            let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.get(d.startLineNumber - 1 - 1);
+            for (let i = 0; i < d.stateDeltas.length; i++) {
+                const delta = d.stateDeltas[i];
+                const state = applyStateStackDiff(prevState, delta);
+                this._states.set(d.startLineNumber + i - 1, state);
+                const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
+                if (offset !== undefined) {
+                    this._backgroundTokenizationStore.setEndState(offset + 1, state);
+                }
+                if (d.startLineNumber + i >= this._model.getLineCount() - 1) {
+                    this._backgroundTokenizationStore.backgroundTokenizationFinished();
+                }
+                prevState = state;
+            }
+        }
+    }
+}
+function lineArrayEditFromModelContentChange(c) {
+    return (
+         (new ArrayEdit(c.map((c) => ( (new SingleArrayEdit(
+            c.range.startLineNumber - 1,
+            c.range.endLineNumber - c.range.startLineNumber,
+            countEOL(c.text)[0]
+        ))))))
+    );
+}
+let TextMateWorkerHost = class TextMateWorkerHost {
+    constructor(_extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService) {
+        this._extensionResourceLoaderService = _extensionResourceLoaderService;
+        this._modelService = _modelService;
+        this._languageConfigurationService = _languageConfigurationService;
+        this._configurationService = _configurationService;
+        this._languageService = _languageService;
+        this._environmentService = _environmentService;
+        this._workerProxyPromise = null;
+        this._worker = null;
+        this._workerProxy = null;
+        this._workerTokenizerControllers = ( (new Map()));
+        this._currentTheme = null;
+        this._currentTokenColorMap = null;
+        this._grammarDefinitions = [];
+    }
+    setGrammarDefinitions(grammarDefinitions) {
+        this._grammarDefinitions = grammarDefinitions;
+        this._killWorker();
+    }
+    dispose() {
+        this._killWorker();
+    }
+    acceptTheme(theme, colorMap) {
+        this._currentTheme = theme;
+        this._currentTokenColorMap = colorMap;
+        if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
+            this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
+        }
+    }
+    getWorkerProxy() {
+        if (!this._workerProxyPromise) {
+            this._workerProxyPromise = this.createWorkerProxy();
+        }
+        return this._workerProxyPromise;
+    }
+    async createWorkerProxy() {
+        const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
+        const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
+        const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
+        const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
+        const useAsar = this._environmentService.isBuilt && !isWeb;
+        const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
+        const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
+        const textmateMain = `${textmateLocation}/release/main.js`;
+        const onigurumaMain = `${onigurumaLocation}/release/main.js`;
+        const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
+        const uri = ( (FileAccess.asBrowserUri(textmateMain).toString(true)));
+        const createData = {
+            grammarDefinitions: this._grammarDefinitions,
+            textmateMainUri: uri,
+            onigurumaMainUri: ( (FileAccess.asBrowserUri(onigurumaMain).toString(true))),
+            onigurumaWASMUri: ( (FileAccess.asBrowserUri(onigurumaWASM).toString(true))),
+        };
+        const worker = createWebWorker(this._modelService, this._languageConfigurationService, {
+            createData,
+            label: 'textMateWorker',
+            moduleId: 'vs/workbench/services/textMate/browser/worker/textMate.worker',
+            host: this,
+        });
+        this._worker = worker;
+        const proxy = await worker.getProxy();
+        if (this._worker !== worker) {
+            return null;
+        }
+        this._workerProxy = proxy;
+        if (this._currentTheme && this._currentTokenColorMap) {
+            this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
+        }
+        return proxy;
+    }
+    _killWorker() {
+        for (const controller of ( (this._workerTokenizerControllers.values()))) {
+            controller.dispose();
+        }
+        this._workerTokenizerControllers.clear();
+        if (this._worker) {
+            this._worker.dispose();
+            this._worker = null;
+        }
+        this._workerProxy = null;
+        this._workerProxyPromise = null;
+    }
+    createBackgroundTokenizer(textModel, tokenStore) {
+        if (this._workerTokenizerControllers.has(( (textModel.uri.toString())))) {
+            throw new BugIndicatingError();
+        }
+        const shouldTokenizeAsync = this._configurationService.getValue('editor.experimental.asyncTokenization');
+        if (shouldTokenizeAsync !== true) {
+            return undefined;
+        }
+        if (textModel.isTooLargeForSyncing()) {
+            return undefined;
+        }
+        const store = ( (new DisposableStore()));
+        this.getWorkerProxy().then((workerProxy) => {
+            if (store.isDisposed || !workerProxy) {
+                return;
+            }
+            store.add(keepAliveWhenAttached(textModel, () => {
+                const controller = ( (new TextMateWorkerTokenizerController(
+                    textModel,
+                    workerProxy,
+                    this._languageService.languageIdCodec,
+                    tokenStore,
+                    INITIAL
+                )));
+                this._workerTokenizerControllers.set(( (textModel.uri.toString())), controller);
+                return toDisposable(() => {
+                    this._workerTokenizerControllers.delete(( (textModel.uri.toString())));
+                    controller.dispose();
+                });
+            }));
+        });
+        return {
+            dispose() {
+                store.dispose();
             },
-            required: ['scopeName', 'path']
+            requestTokens: (startLineNumber, endLineNumberExclusive) => {
+                this.getWorkerProxy().then((workerProxy) => {
+                    workerProxy?.retokenize(( (textModel.uri.toString())), startLineNumber, endLineNumberExclusive);
+                });
+            },
+        };
+    }
+    async readFile(_resource) {
+        const resource = URI.revive(_resource);
+        return this._extensionResourceLoaderService.readExtensionResource(resource);
+    }
+    async setTokensAndStates(_resource, versionId, tokens, lineEndStateDeltas) {
+        const resource = URI.revive(_resource);
+        const controller = this._workerTokenizerControllers.get(( (resource.toString())));
+        if (controller) {
+            controller.setTokensAndStates(versionId, tokens, lineEndStateDeltas);
         }
     }
-})));
-class TMScopeRegistry extends Disposable {
+};
+TextMateWorkerHost = ( (__decorate([
+    ( (__param(0, IExtensionResourceLoaderService))),
+    ( (__param(1, IModelService))),
+    ( (__param(2, ILanguageConfigurationService))),
+    ( (__param(3, IConfigurationService))),
+    ( (__param(4, ILanguageService))),
+    ( (__param(5, IEnvironmentService)))
+], TextMateWorkerHost)));
+function keepAliveWhenAttached(textModel, factory) {
+    const disposableStore = ( (new DisposableStore()));
+    const subStore = disposableStore.add(( (new DisposableStore())));
+    function checkAttached() {
+        if (textModel.isAttachedToEditor()) {
+            subStore.add(factory());
+        }
+        else {
+            subStore.clear();
+        }
+    }
+    checkAttached();
+    disposableStore.add(textModel.onDidChangeAttached(() => {
+        checkAttached();
+    }));
+    return disposableStore;
+}
+class TMScopeRegistry {
     constructor() {
-        super();
         this._scopeNameToLanguageRegistration = Object.create(null);
     }
     reset() {
@@ -145,7 +491,7 @@ class TMGrammarFactory extends Disposable {
         super();
         this._host = host;
         this._initialState = vscodeTextmate.INITIAL;
-        this._scopeRegistry = this._register(( (new TMScopeRegistry())));
+        this._scopeRegistry = ( (new TMScopeRegistry()));
         this._injections = {};
         this._injectedEmbeddedLanguages = {};
         this._languageToScope = ( (new Map()));
@@ -236,73 +582,114 @@ class TMGrammarFactory extends Disposable {
                 embeddedLanguages,
                 tokenTypes: grammarDefinition.tokenTypes,
                 balancedBracketSelectors: grammarDefinition.balancedBracketSelectors,
-                unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors,
-            });
-        }
-        catch (err) {
-            if (err.message && err.message.startsWith('No grammar provided for')) {
-                throw new Error(missingTMGrammarErrorMessage);
-            }
-            throw err;
-        }
-        return {
-            languageId: languageId,
-            grammar: grammar,
-            initialState: this._initialState,
-            containsEmbeddedLanguages: containsEmbeddedLanguages
-        };
-    }
-}
-class TMTokenization extends Disposable {
-    constructor(grammar, initialState, containsEmbeddedLanguages) {
-        super();
-        this._onDidEncounterLanguage = this._register(( (new Emitter())));
-        this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
-        this._grammar = grammar;
-        this._initialState = initialState;
-        this._containsEmbeddedLanguages = containsEmbeddedLanguages;
-        this._seenLanguages = [];
-    }
-    getInitialState() {
-        return this._initialState;
-    }
-    tokenize(line, hasEOL, state) {
-        throw new Error('Not supported!');
-    }
-    tokenizeEncoded(line, hasEOL, state) {
-        const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
-        if (textMateResult.stoppedEarly) {
-            console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
-            return (
-                 (new EncodedTokenizationResult(textMateResult.tokens, state))
-            );
-        }
-        if (this._containsEmbeddedLanguages) {
-            const seenLanguages = this._seenLanguages;
-            const tokens = textMateResult.tokens;
-            for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
-                const metadata = tokens[(i << 1) + 1];
-                const languageId = TokenMetadata.getLanguageId(metadata);
-                if (!seenLanguages[languageId]) {
-                    seenLanguages[languageId] = true;
-                    this._onDidEncounterLanguage.fire(languageId);
-                }
-            }
-        }
-        let endState;
-        if (state.equals(textMateResult.ruleStack)) {
-            endState = state;
+                unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors,
+            });
         }
-        else {
-            endState = textMateResult.ruleStack;
+        catch (err) {
+            if (err.message && err.message.startsWith('No grammar provided for')) {
+                throw new Error(missingTMGrammarErrorMessage);
+            }
+            throw err;
         }
-        return (
-             (new EncodedTokenizationResult(textMateResult.tokens, endState))
-        );
+        return {
+            languageId: languageId,
+            grammar: grammar,
+            initialState: this._initialState,
+            containsEmbeddedLanguages: containsEmbeddedLanguages
+        };
     }
 }
-let AbstractTextMateService = class AbstractTextMateService extends Disposable {
-    constructor(_languageService, _themeService, _extensionResourceLoaderService, _notificationService, _logService, _configurationService, _progressService) {
+const grammarsExtPoint = ( (ExtensionsRegistry.registerExtensionPoint({
+    extensionPoint: 'grammars',
+    deps: [languagesExtPoint],
+    jsonSchema: {
+        description: ( (localize(
+            'vscode.extension.contributes.grammars',
+            'Contributes textmate tokenizers.'
+        ))),
+        type: 'array',
+        defaultSnippets: [{ body: [{ language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' }] }],
+        items: {
+            type: 'object',
+            defaultSnippets: [{ body: { language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' } }],
+            properties: {
+                language: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.language',
+                        'Language identifier for which this syntax is contributed to.'
+                    ))),
+                    type: 'string'
+                },
+                scopeName: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.scopeName',
+                        'Textmate scope name used by the tmLanguage file.'
+                    ))),
+                    type: 'string'
+                },
+                path: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.path',
+                        'Path of the tmLanguage file. The path is relative to the extension folder and typically starts with \'./syntaxes/\'.'
+                    ))),
+                    type: 'string'
+                },
+                embeddedLanguages: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.embeddedLanguages',
+                        'A map of scope name to language id if this grammar contains embedded languages.'
+                    ))),
+                    type: 'object'
+                },
+                tokenTypes: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.tokenTypes',
+                        'A map of scope name to token types.'
+                    ))),
+                    type: 'object',
+                    additionalProperties: {
+                        enum: ['string', 'comment', 'other']
+                    }
+                },
+                injectTo: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.injectTo',
+                        'List of language scope names to which this grammar is injected to.'
+                    ))),
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                balancedBracketScopes: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.balancedBracketScopes',
+                        'Defines which scope names contain balanced brackets.'
+                    ))),
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    },
+                    default: ['*'],
+                },
+                unbalancedBracketScopes: {
+                    description: ( (localize(
+                        'vscode.extension.contributes.grammars.unbalancedBracketScopes',
+                        'Defines which scope names do not contain balanced brackets.'
+                    ))),
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    },
+                    default: [],
+                },
+            },
+            required: ['scopeName', 'path']
+        }
+    }
+})));
+let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
+    constructor(_languageService, _themeService, _extensionResourceLoaderService, _notificationService, _logService, _configurationService, _progressService, _environmentService, _instantiationService) {
         super();
         this._languageService = _languageService;
         this._themeService = _themeService;
@@ -311,108 +698,120 @@ let AbstractTextMateService = class AbstractTextMateService extends Disposable {
         this._logService = _logService;
         this._configurationService = _configurationService;
         this._progressService = _progressService;
+        this._environmentService = _environmentService;
+        this._instantiationService = _instantiationService;
         this._onDidEncounterLanguage = this._register(( (new Emitter())));
         this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
-        this._vscodeOniguruma = null;
-        this._styleElement = createStyleSheet();
-        this._styleElement.className = 'vscode-tokens-styles';
         this._createdModes = [];
         this._encounteredLanguages = [];
         this._debugMode = false;
         this._debugModePrintFunc = () => { };
         this._grammarDefinitions = null;
         this._grammarFactory = null;
-        this._tokenizersRegistrations = [];
+        this._tokenizersRegistrations = ( (new DisposableStore()));
         this._currentTheme = null;
         this._currentTokenColorMap = null;
-        grammarsExtPoint.setHandler((extensions) => {
-            this._grammarDefinitions = null;
-            if (this._grammarFactory) {
-                this._grammarFactory.dispose();
-                this._grammarFactory = null;
-                this._onDidDisposeGrammarFactory();
-            }
-            this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
-            this._grammarDefinitions = [];
-            for (const extension of extensions) {
-                const grammars = extension.value;
-                for (const grammar of grammars) {
-                    if (!this._validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector)) {
-                        continue;
-                    }
-                    const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
-                    const embeddedLanguages = Object.create(null);
-                    if (grammar.embeddedLanguages) {
-                        const scopes = ( (Object.keys(grammar.embeddedLanguages)));
-                        for (let i = 0, len = scopes.length; i < len; i++) {
-                            const scope = scopes[i];
-                            const language = grammar.embeddedLanguages[scope];
-                            if (typeof language !== 'string') {
-                                continue;
-                            }
-                            if (this._languageService.isRegisteredLanguageId(language)) {
-                                embeddedLanguages[scope] = this._languageService.languageIdCodec.encodeLanguageId(language);
-                            }
-                        }
-                    }
-                    const tokenTypes = Object.create(null);
-                    if (grammar.tokenTypes) {
-                        const scopes = ( (Object.keys(grammar.tokenTypes)));
-                        for (const scope of scopes) {
-                            const tokenType = grammar.tokenTypes[scope];
-                            switch (tokenType) {
-                                case 'string':
-                                    tokenTypes[scope] = 2 ;
-                                    break;
-                                case 'other':
-                                    tokenTypes[scope] = 0 ;
-                                    break;
-                                case 'comment':
-                                    tokenTypes[scope] = 1 ;
-                                    break;
-                            }
-                        }
-                    }
-                    let validLanguageId = null;
-                    if (grammar.language && this._languageService.isRegisteredLanguageId(grammar.language)) {
-                        validLanguageId = grammar.language;
-                    }
-                    function asStringArray(array, defaultValue) {
-                        if (!Array.isArray(array)) {
-                            return defaultValue;
-                        }
-                        if (!array.every(e => typeof e === 'string')) {
-                            return defaultValue;
-                        }
-                        return array;
-                    }
-                    this._grammarDefinitions.push({
-                        location: grammarLocation,
-                        language: validLanguageId ? validLanguageId : undefined,
-                        scopeName: grammar.scopeName,
-                        embeddedLanguages: embeddedLanguages,
-                        tokenTypes: tokenTypes,
-                        injectTo: grammar.injectTo,
-                        balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
-                        unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
-                    });
-                    if (validLanguageId) {
-                        this._tokenizersRegistrations.push(TokenizationRegistry.registerFactory(validLanguageId, this._createFactory(validLanguageId)));
-                    }
-                }
-            }
-            for (const createMode of this._createdModes) {
-                TokenizationRegistry.getOrCreate(createMode);
-            }
-        });
-        this._updateTheme(this._grammarFactory, this._themeService.getColorTheme(), true);
+        this._workerHost = this._instantiationService.createInstance(TextMateWorkerHost);
+        this._vscodeOniguruma = null;
+        this._styleElement = createStyleSheet();
+        this._styleElement.className = 'vscode-tokens-styles';
+        grammarsExtPoint.setHandler((extensions) => this.handleGrammarsExtPoint(extensions));
+        this._updateTheme(this._themeService.getColorTheme(), true);
         this._register(this._themeService.onDidColorThemeChange(() => {
-            this._updateTheme(this._grammarFactory, this._themeService.getColorTheme(), false);
+            this._updateTheme(this._themeService.getColorTheme(), false);
         }));
         this._languageService.onDidEncounterLanguage((languageId) => {
             this._createdModes.push(languageId);
         });
     }
+    handleGrammarsExtPoint(extensions) {
+        this._grammarDefinitions = null;
+        if (this._grammarFactory) {
+            this._grammarFactory.dispose();
+            this._grammarFactory = null;
+        }
+        this._tokenizersRegistrations.clear();
+        this._grammarDefinitions = [];
+        for (const extension of extensions) {
+            const grammars = extension.value;
+            for (const grammar of grammars) {
+                const def = this.createValidGrammarDefinition(extension, grammar);
+                if (def) {
+                    this._grammarDefinitions.push(def);
+                    if (def.language) {
+                        this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(def.language, {
+                            createTokenizationSupport: async () => this.createTokenizationSupport(def.language)
+                        }));
+                    }
+                }
+            }
+        }
+        this._workerHost.setGrammarDefinitions(this._grammarDefinitions);
+        for (const createdMode of this._createdModes) {
+            TokenizationRegistry.getOrCreate(createdMode);
+        }
+    }
+    createValidGrammarDefinition(extension, grammar) {
+        if (!validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector, this._languageService)) {
+            return null;
+        }
+        const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
+        const embeddedLanguages = Object.create(null);
+        if (grammar.embeddedLanguages) {
+            const scopes = ( (Object.keys(grammar.embeddedLanguages)));
+            for (let i = 0, len = scopes.length; i < len; i++) {
+                const scope = scopes[i];
+                const language = grammar.embeddedLanguages[scope];
+                if (typeof language !== 'string') {
+                    continue;
+                }
+                if (this._languageService.isRegisteredLanguageId(language)) {
+                    embeddedLanguages[scope] = this._languageService.languageIdCodec.encodeLanguageId(language);
+                }
+            }
+        }
+        const tokenTypes = Object.create(null);
+        if (grammar.tokenTypes) {
+            const scopes = ( (Object.keys(grammar.tokenTypes)));
+            for (const scope of scopes) {
+                const tokenType = grammar.tokenTypes[scope];
+                switch (tokenType) {
+                    case 'string':
+                        tokenTypes[scope] = 2 ;
+                        break;
+                    case 'other':
+                        tokenTypes[scope] = 0 ;
+                        break;
+                    case 'comment':
+                        tokenTypes[scope] = 1 ;
+                        break;
+                }
+            }
+        }
+        let validLanguageId = null;
+        if (grammar.language && this._languageService.isRegisteredLanguageId(grammar.language)) {
+            validLanguageId = grammar.language;
+        }
+        function asStringArray(array, defaultValue) {
+            if (!Array.isArray(array)) {
+                return defaultValue;
+            }
+            if (!array.every(e => typeof e === 'string')) {
+                return defaultValue;
+            }
+            return array;
+        }
+        return {
+            location: grammarLocation,
+            language: validLanguageId || undefined,
+            scopeName: grammar.scopeName,
+            embeddedLanguages: embeddedLanguages,
+            tokenTypes: tokenTypes,
+            injectTo: grammar.injectTo,
+            balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
+            unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
+        };
+    }
     startDebugMode(printFn, onStop) {
         if (this._debugMode) {
             this._notificationService.error(( (localize('alreadyDebugging', "Already Logging."))));
@@ -451,7 +850,7 @@ let AbstractTextMateService = class AbstractTextMateService extends Disposable {
         }
     }
     _canCreateGrammarFactory() {
-        return (this._grammarDefinitions ? true : false);
+        return !!this._grammarDefinitions;
     }
     async _getOrCreateGrammarFactory() {
         if (this._grammarFactory) {
@@ -470,162 +869,66 @@ let AbstractTextMateService = class AbstractTextMateService extends Disposable {
             logError: (msg, err) => this._logService.error(msg, err),
             readFile: (resource) => this._extensionResourceLoaderService.readExtensionResource(resource)
         }, this._grammarDefinitions || [], vscodeTextmate, onigLib)));
-        this._onDidCreateGrammarFactory(this._grammarDefinitions || []);
-        this._updateTheme(this._grammarFactory, this._themeService.getColorTheme(), true);
+        this._updateTheme(this._themeService.getColorTheme(), true);
         return this._grammarFactory;
     }
-    _createFactory(languageId) {
-        return {
-            createTokenizationSupport: async () => {
-                if (!this._languageService.isRegisteredLanguageId(languageId)) {
-                    return null;
-                }
-                if (!this._canCreateGrammarFactory()) {
-                    return null;
-                }
-                const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
-                try {
-                    const grammarFactory = await this._getOrCreateGrammarFactory();
-                    if (!grammarFactory.has(languageId)) {
-                        return null;
-                    }
-                    const r = await grammarFactory.createGrammar(languageId, encodedLanguageId);
-                    if (!r.grammar) {
-                        return null;
-                    }
-                    const tokenization = ( (new TMTokenization(r.grammar, r.initialState, r.containsEmbeddedLanguages)));
-                    tokenization.onDidEncounterLanguage((encodedLanguageId) => {
-                        if (!this._encounteredLanguages[encodedLanguageId]) {
-                            const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
-                            this._encounteredLanguages[encodedLanguageId] = true;
-                            this._onDidEncounterLanguage.fire(languageId);
-                        }
-                    });
-                    return (
-                         (new TMTokenizationSupportWithLineLimit(languageId, encodedLanguageId, tokenization, this._configurationService))
-                    );
-                }
-                catch (err) {
-                    if (err.message && err.message === missingTMGrammarErrorMessage) {
-                        return null;
-                    }
-                    onUnexpectedError(err);
-                    return null;
+    async createTokenizationSupport(languageId) {
+        if (!this._languageService.isRegisteredLanguageId(languageId)) {
+            return null;
+        }
+        if (!this._canCreateGrammarFactory()) {
+            return null;
+        }
+        try {
+            const grammarFactory = await this._getOrCreateGrammarFactory();
+            if (!grammarFactory.has(languageId)) {
+                return null;
+            }
+            const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
+            const r = await grammarFactory.createGrammar(languageId, encodedLanguageId);
+            if (!r.grammar) {
+                return null;
+            }
+            const tokenization = ( (new TextMateTokenizationSupport(
+                r.grammar,
+                r.initialState,
+                r.containsEmbeddedLanguages,
+                (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore)
+            )));
+            tokenization.onDidEncounterLanguage((encodedLanguageId) => {
+                if (!this._encounteredLanguages[encodedLanguageId]) {
+                    const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
+                    this._encounteredLanguages[encodedLanguageId] = true;
+                    this._onDidEncounterLanguage.fire(languageId);
                 }
+            });
+            return (
+                 (new TokenizationSupportWithLineLimit(languageId, encodedLanguageId, tokenization, this._configurationService))
+            );
+        }
+        catch (err) {
+            if (err.message && err.message === missingTMGrammarErrorMessage) {
+                return null;
             }
-        };
-    }
-    static _toColorMap(colorMap) {
-        const result = [null];
-        for (let i = 1, len = colorMap.length; i < len; i++) {
-            result[i] = ( (Color.fromHex(colorMap[i])));
+            onUnexpectedError(err);
+            return null;
         }
-        return result;
     }
-    _updateTheme(grammarFactory, colorTheme, forceUpdate) {
-        if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && AbstractTextMateService.equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors) && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
+    _updateTheme(colorTheme, forceUpdate) {
+        if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
+            && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
             return;
         }
         this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
         this._currentTokenColorMap = colorTheme.tokenColorMap;
-        this._doUpdateTheme(grammarFactory, this._currentTheme, this._currentTokenColorMap);
-    }
-    _doUpdateTheme(grammarFactory, theme, tokenColorMap) {
-        grammarFactory?.setTheme(theme, tokenColorMap);
-        const colorMap = AbstractTextMateService._toColorMap(tokenColorMap);
+        this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
+        const colorMap = toColorMap(this._currentTokenColorMap);
         const cssRules = generateTokensCSSForColorMap(colorMap);
         this._styleElement.textContent = cssRules;
         TokenizationRegistry.setColorMap(colorMap);
-    }
-    static equalsTokenRules(a, b) {
-        if (!b || !a || b.length !== a.length) {
-            return false;
-        }
-        for (let i = b.length - 1; i >= 0; i--) {
-            const r1 = b[i];
-            const r2 = a[i];
-            if (r1.scope !== r2.scope) {
-                return false;
-            }
-            const s1 = r1.settings;
-            const s2 = r2.settings;
-            if (s1 && s2) {
-                if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
-                    return false;
-                }
-            }
-            else if (!s1 || !s2) {
-                return false;
-            }
-        }
-        return true;
-    }
-    _validateGrammarExtensionPoint(extensionLocation, syntax, collector) {
-        if (syntax.language && ((typeof syntax.language !== 'string') || !this._languageService.isRegisteredLanguageId(syntax.language))) {
-            collector.error(( (localize(
-                'invalid.language',
-                "Unknown language in `contributes.{0}.language`. Provided value: {1}",
-                grammarsExtPoint.name,
-                String(syntax.language)
-            ))));
-            return false;
-        }
-        if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
-            collector.error(( (localize(
-                'invalid.scopeName',
-                "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
-                grammarsExtPoint.name,
-                String(syntax.scopeName)
-            ))));
-            return false;
-        }
-        if (!syntax.path || (typeof syntax.path !== 'string')) {
-            collector.error(( (localize(
-                'invalid.path.0',
-                "Expected string in `contributes.{0}.path`. Provided value: {1}",
-                grammarsExtPoint.name,
-                String(syntax.path)
-            ))));
-            return false;
-        }
-        if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
-            collector.error(( (localize(
-                'invalid.injectTo',
-                "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
-                grammarsExtPoint.name,
-                JSON.stringify(syntax.injectTo)
-            ))));
-            return false;
-        }
-        if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
-            collector.error(( (localize(
-                'invalid.embeddedLanguages',
-                "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
-                grammarsExtPoint.name,
-                JSON.stringify(syntax.embeddedLanguages)
-            ))));
-            return false;
-        }
-        if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
-            collector.error(( (localize(
-                'invalid.tokenTypes',
-                "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
-                grammarsExtPoint.name,
-                JSON.stringify(syntax.tokenTypes)
-            ))));
-            return false;
-        }
-        const grammarLocation = joinPath(extensionLocation, syntax.path);
-        if (!isEqualOrParent(grammarLocation, extensionLocation)) {
-            collector.warn(( (localize(
-                'invalid.path.1',
-                "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
-                grammarsExtPoint.name,
-                grammarLocation.path,
-                extensionLocation.path
-            ))));
+        if (this._currentTheme && this._currentTokenColorMap) {
+            this._workerHost.acceptTheme(this._currentTheme, this._currentTokenColorMap);
         }
-        return true;
     }
     async createGrammar(languageId) {
         if (!this._languageService.isRegisteredLanguageId(languageId)) {
@@ -639,82 +942,151 @@ let AbstractTextMateService = class AbstractTextMateService extends Disposable {
         const { grammar } = await grammarFactory.createGrammar(languageId, encodedLanguageId);
         return grammar;
     }
-    _onDidCreateGrammarFactory(grammarDefinitions) {
-    }
-    _onDidDisposeGrammarFactory() {
-    }
     _getVSCodeOniguruma() {
         if (!this._vscodeOniguruma) {
-            this._vscodeOniguruma = this._doGetVSCodeOniguruma();
+            this._vscodeOniguruma = (async () => {
+                const [vscodeOniguruma, wasm] = await Promise.all([import('vscode-oniguruma').then(module => module.default ?? module), this._loadVSCodeOnigurumaWASM()]);
+                await vscodeOniguruma.loadWASM({
+                    data: wasm,
+                    print: (str) => {
+                        this._debugModePrintFunc(str);
+                    }
+                });
+                return vscodeOniguruma;
+            })();
         }
         return this._vscodeOniguruma;
     }
-    async _doGetVSCodeOniguruma() {
-        const [vscodeOniguruma, wasm] = await Promise.all([import('vscode-oniguruma').then(module => module.default ?? module), this._loadVSCodeOnigurumWASM()]);
-        const options = {
-            data: wasm,
-            print: (str) => {
-                this._debugModePrintFunc(str);
-            }
-        };
-        await vscodeOniguruma.loadWASM(options);
-        return vscodeOniguruma;
+    async _loadVSCodeOnigurumaWASM() {
+        if (isWeb) {
+            const response = await fetch(( (FileAccess.asBrowserUri('vscode-oniguruma/../onig.wasm').toString(true))));
+            return await response.arrayBuffer();
+        }
+        else {
+            const response = await fetch(this._environmentService.isBuilt
+                ? ( (FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`).toString(true)))
+                : ( (FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`).toString(true))));
+            return response;
+        }
     }
 };
-AbstractTextMateService = ( (__decorate([
+TextMateTokenizationFeature = ( (__decorate([
     ( (__param(0, ILanguageService))),
     ( (__param(1, IWorkbenchThemeService))),
     ( (__param(2, IExtensionResourceLoaderService))),
     ( (__param(3, INotificationService))),
     ( (__param(4, ILogService))),
     ( (__param(5, IConfigurationService))),
-    ( (__param(6, IProgressService)))
-], AbstractTextMateService)));
-let TMTokenizationSupportWithLineLimit = class TMTokenizationSupportWithLineLimit {
-    constructor(languageId, encodedLanguageId, actual, _configurationService) {
-        this._configurationService = _configurationService;
-        this._languageId = languageId;
-        this._encodedLanguageId = encodedLanguageId;
-        this._actual = actual;
-        this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
-            overrideIdentifier: this._languageId
-        });
-        this._configurationService.onDidChangeConfiguration(e => {
-            if (e.affectsConfiguration('editor.maxTokenizationLineLength')) {
-                this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', {
-                    overrideIdentifier: this._languageId
-                });
+    ( (__param(6, IProgressService))),
+    ( (__param(7, IWorkbenchEnvironmentService))),
+    ( (__param(8, IInstantiationService)))
+], TextMateTokenizationFeature)));
+function toColorMap(colorMap) {
+    const result = [null];
+    for (let i = 1, len = colorMap.length; i < len; i++) {
+        result[i] = ( (Color.fromHex(colorMap[i])));
+    }
+    return result;
+}
+function equalsTokenRules(a, b) {
+    if (!b || !a || b.length !== a.length) {
+        return false;
+    }
+    for (let i = b.length - 1; i >= 0; i--) {
+        const r1 = b[i];
+        const r2 = a[i];
+        if (r1.scope !== r2.scope) {
+            return false;
+        }
+        const s1 = r1.settings;
+        const s2 = r2.settings;
+        if (s1 && s2) {
+            if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
+                return false;
             }
-        });
+        }
+        else if (!s1 || !s2) {
+            return false;
+        }
     }
-    getInitialState() {
-        return this._actual.getInitialState();
+    return true;
+}
+function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
+    if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
+        collector.error(( (localize(
+            'invalid.language',
+            "Unknown language in `contributes.{0}.language`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.language)
+        ))));
+        return false;
     }
-    tokenize(line, hasEOL, state) {
-        throw new Error('Not supported!');
+    if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
+        collector.error(( (localize(
+            'invalid.scopeName',
+            "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.scopeName)
+        ))));
+        return false;
     }
-    tokenizeEncoded(line, hasEOL, state) {
-        if (line.length >= this._maxTokenizationLineLength) {
-            return nullTokenizeEncoded(this._encodedLanguageId, state);
-        }
-        return this._actual.tokenizeEncoded(line, hasEOL, state);
+    if (!syntax.path || (typeof syntax.path !== 'string')) {
+        collector.error(( (localize(
+            'invalid.path.0',
+            "Expected string in `contributes.{0}.path`. Provided value: {1}",
+            grammarsExtPoint.name,
+            String(syntax.path)
+        ))));
+        return false;
     }
-};
-TMTokenizationSupportWithLineLimit = ( (__decorate([
-    ( (__param(3, IConfigurationService)))
-], TMTokenizationSupportWithLineLimit)));
-let TextMateService = class TextMateService extends createInjectedClass(AbstractTextMateService) {
-    constructor(loadVSCodeOnigurumWASM, instantiationService) {
-        super(instantiationService);
-        this.loadVSCodeOnigurumWASM = loadVSCodeOnigurumWASM;
+    if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
+        collector.error(( (localize(
+            'invalid.injectTo',
+            "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.injectTo)
+        ))));
+        return false;
+    }
+    if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
+        collector.error(( (localize(
+            'invalid.embeddedLanguages',
+            "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.embeddedLanguages)
+        ))));
+        return false;
     }
-    _loadVSCodeOnigurumWASM() {
-        return this.loadVSCodeOnigurumWASM();
+    if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
+        collector.error(( (localize(
+            'invalid.tokenTypes',
+            "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
+            grammarsExtPoint.name,
+            JSON.stringify(syntax.tokenTypes)
+        ))));
+        return false;
+    }
+    const grammarLocation = joinPath(extensionLocation, syntax.path);
+    if (!isEqualOrParent(grammarLocation, extensionLocation)) {
+        collector.warn(( (localize(
+            'invalid.path.1',
+            "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
+            grammarsExtPoint.name,
+            grammarLocation.path,
+            extensionLocation.path
+        ))));
+    }
+    return true;
+}
+let _TextMateTokenizationFeature = class _TextMateTokenizationFeature extends createInjectedClass(TextMateTokenizationFeature) {
+    constructor(loadVSCodeOnigurumaWASM, instantiationService) {
+        super(instantiationService);
+        this['_loadVSCodeOnigurumaWASM'] = loadVSCodeOnigurumaWASM;
     }
 };
-TextMateService = ( __decorate([
+_TextMateTokenizationFeature = ( __decorate([
     ( __param(1, IInstantiationService))
-], TextMateService));
+], _TextMateTokenizationFeature));
 const extensionPoint = getExtensionPoint('grammars');
 function setGrammars(grammars, getContent, extension = Services.get().extension ?? DEFAULT_EXTENSION) {
     extensionPoint.acceptUsers([{
@@ -727,13 +1099,13 @@ function setGrammars(grammars, getContent, extension = Services.get().extension
     }
 }
 function initialize(instantiationService) {
-    instantiationService.invokeFunction((accessor) => accessor.get(ITextMateService));
+    instantiationService.invokeFunction((accessor) => accessor.get(ITextMateTokenizationService));
 }
 function getServiceOverride(getOnigLib) {
     onServicesInitialized(initialize);
     return {
         ...getServiceOverride$1(),
-        [( ITextMateService.toString())]: new SyncDescriptor(TextMateService, [getOnigLib])
+        [( ITextMateTokenizationService.toString())]: new SyncDescriptor(_TextMateTokenizationFeature, [getOnigLib])
     };
 }
-export { ITextMateService, getServiceOverride as default, setGrammars };
+export { ITextMateTokenizationService, getServiceOverride as default, setGrammars };