@codingame/monaco-vscode-api 1.82.4-next.1 → 1.82.5-next.0

package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js

@@ -1,157 +0,0 @@
-import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
-import { keepAlive, autorun, observableFromEvent } from 'monaco-editor/esm/vs/base/common/observable.js';
-import { countEOL } from 'monaco-editor/esm/vs/editor/common/core/eolCounter.js';
-import { LineRange } from 'monaco-editor/esm/vs/editor/common/core/lineRange.js';
-import { Range } from 'monaco-editor/esm/vs/editor/common/core/range.js';
-import { TokenizationStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
-import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
-import { MonotonousIndexTransformer, ArrayEdit, SingleArrayEdit } from '../arrayOperation.js';
-import { applyStateStackDiff, INITIAL } from 'vscode-textmate';
-class TextMateWorkerTokenizerController extends Disposable {
-    static { this._id = 0; }
-    constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _configurationService, _maxTokenizationLineLength) {
-        super();
-        this._model = _model;
-        this._worker = _worker;
-        this._languageIdCodec = _languageIdCodec;
-        this._backgroundTokenizationStore = _backgroundTokenizationStore;
-        this._configurationService = _configurationService;
-        this._maxTokenizationLineLength = _maxTokenizationLineLength;
-        this.controllerId = TextMateWorkerTokenizerController._id++;
-        this._pendingChanges = [];
-        this._states = ( new TokenizationStateStore());
-        this._loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
-        this._register(keepAlive(this._loggingEnabled));
-        this._register(this._model.onDidChangeContent((e) => {
-            if (this._shouldLog) {
-                console.log('model change', {
-                    fileName: this._model.uri.fsPath.split('\\').pop(),
-                    changes: changesToString(e.changes),
-                });
-            }
-            this._worker.acceptModelChanged(this.controllerId, e);
-            this._pendingChanges.push(e);
-        }));
-        this._register(this._model.onDidChangeLanguage((e) => {
-            const languageId = this._model.getLanguageId();
-            const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
-            this._worker.acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
-        }));
-        const languageId = this._model.getLanguageId();
-        const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
-        this._worker.acceptNewModel({
-            uri: this._model.uri,
-            versionId: this._model.getVersionId(),
-            lines: this._model.getLinesContent(),
-            EOL: this._model.getEOL(),
-            languageId,
-            encodedLanguageId,
-            maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
-            controllerId: this.controllerId,
-        });
-        this._register(autorun(reader => {
-            const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
-            this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
-        }));
-    }
-    dispose() {
-        super.dispose();
-        this._worker.acceptRemovedModel(this.controllerId);
-    }
-    requestTokens(startLineNumber, endLineNumberExclusive) {
-        this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
-    }
-    async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
-        if (this.controllerId !== controllerId) {
-            return;
-        }
-        let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
-        if (this._shouldLog) {
-            console.log('received background tokenization result', {
-                fileName: this._model.uri.fsPath.split('\\').pop(),
-                updatedTokenLines: ( tokens.map((t) => t.getLineRange())).join(' & '),
-                updatedStateLines: ( stateDeltas.map((s) => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(' & '),
-            });
-        }
-        if (this._shouldLog) {
-            const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(' then ');
-            console.log('Applying changes to local states', changes);
-        }
-        while (this._pendingChanges.length > 0 &&
-            this._pendingChanges[0].versionId <= versionId) {
-            const change = this._pendingChanges.shift();
-            this._states.acceptChanges(change.changes);
-        }
-        if (this._pendingChanges.length > 0) {
-            if (this._shouldLog) {
-                const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(' then ');
-                console.log('Considering non-processed changes', changes);
-            }
-            const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
-            const b = ( new ContiguousMultilineTokensBuilder());
-            for (const t of tokens) {
-                for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
-                    const result = curToFutureTransformerTokens.transform(i - 1);
-                    if (result !== undefined) {
-                        b.add(i, t.getLineTokens(i));
-                    }
-                }
-            }
-            tokens = b.finalize();
-            for (const change of this._pendingChanges) {
-                for (const innerChanges of change.changes) {
-                    for (let j = 0; j < tokens.length; j++) {
-                        tokens[j].applyEdit(innerChanges.range, innerChanges.text);
-                    }
-                }
-            }
-        }
-        const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
-        if (!this._applyStateStackDiffFn || !this._initialState) {
-            this._applyStateStackDiffFn = applyStateStackDiff;
-            this._initialState = INITIAL;
-        }
-        for (const d of stateDeltas) {
-            let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.getEndState(d.startLineNumber - 1);
-            for (let i = 0; i < d.stateDeltas.length; i++) {
-                const delta = d.stateDeltas[i];
-                let state;
-                if (delta) {
-                    state = this._applyStateStackDiffFn(prevState, delta);
-                    this._states.setEndState(d.startLineNumber + i, state);
-                }
-                else {
-                    state = this._states.getEndState(d.startLineNumber + i);
-                }
-                const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
-                if (offset !== undefined) {
-                    this._backgroundTokenizationStore.setEndState(offset + 1, state);
-                }
-                if (d.startLineNumber + i >= this._model.getLineCount() - 1) {
-                    this._backgroundTokenizationStore.backgroundTokenizationFinished();
-                }
-                prevState = state;
-            }
-        }
-        this._backgroundTokenizationStore.setTokens(tokens);
-    }
-    get _shouldLog() { return this._loggingEnabled.get(); }
-}
-function fullLineArrayEditFromModelContentChange(c) {
-    return ( new ArrayEdit(( c.map((c) => ( new SingleArrayEdit(
-        c.range.startLineNumber - 1,
-        c.range.endLineNumber - c.range.startLineNumber + 1,
-        countEOL(c.text)[0] + 1
-    ))))));
-}
-function changesToString(changes) {
-    return ( changes.map(c => ( Range.lift(c.range).toString()) + ' => ' + c.text)).join(' & ');
-}
-function observableConfigValue(key, defaultValue, configurationService) {
-    return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
-        if (e.affectsConfiguration(key)) {
-            handleChange(e);
-        }
-    }), () => configurationService.getValue(key) ?? defaultValue);
-}
-export { TextMateWorkerTokenizerController };

package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js

@@ -1,200 +0,0 @@
-import { __decorate, __param } from '../../../../../../../../external/tslib/tslib.es6.js';
-import { DisposableStore, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
-import { FileAccess, nodeModulesPath, nodeModulesAsarPath } from 'monaco-editor/esm/vs/base/common/network.js';
-import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
-import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
-import { createWebWorker } from 'monaco-editor/esm/vs/editor/browser/services/webWorker.js';
-import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
-import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
-import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
-import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
-import { IEnvironmentService } from 'monaco-editor/esm/vs/platform/environment/common/environment.js';
-import { IExtensionResourceLoaderService } from '../../../../../platform/extensionResourceLoader/common/extensionResourceLoader.js';
-import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
-import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
-import { TextMateWorkerTokenizerController } from './textMateWorkerTokenizerController.js';
-var ThreadedBackgroundTokenizerFactory_1;
-let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
-    static { ThreadedBackgroundTokenizerFactory_1 = this; }
-    static { this._reportedMismatchingTokens = false; }
-    constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
-        this._reportTokenizationTime = _reportTokenizationTime;
-        this._shouldTokenizeAsync = _shouldTokenizeAsync;
-        this._extensionResourceLoaderService = _extensionResourceLoaderService;
-        this._modelService = _modelService;
-        this._languageConfigurationService = _languageConfigurationService;
-        this._configurationService = _configurationService;
-        this._languageService = _languageService;
-        this._environmentService = _environmentService;
-        this._notificationService = _notificationService;
-        this._telemetryService = _telemetryService;
-        this._workerProxyPromise = null;
-        this._worker = null;
-        this._workerProxy = null;
-        this._workerTokenizerControllers = ( new Map());
-        this._currentTheme = null;
-        this._currentTokenColorMap = null;
-        this._grammarDefinitions = [];
-    }
-    dispose() {
-        this._disposeWorker();
-    }
-    createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength) {
-        if (!this._shouldTokenizeAsync() || textModel.isTooLargeForSyncing()) {
-            return undefined;
-        }
-        const store = ( new DisposableStore());
-        const controllerContainer = this._getWorkerProxy().then((workerProxy) => {
-            if (store.isDisposed || !workerProxy) {
-                return undefined;
-            }
-            const controllerContainer = { controller: undefined, worker: this._worker };
-            store.add(keepAliveWhenAttached(textModel, () => {
-                const controller = ( new TextMateWorkerTokenizerController(
-                    textModel,
-                    workerProxy,
-                    this._languageService.languageIdCodec,
-                    tokenStore,
-                    this._configurationService,
-                    maxTokenizationLineLength
-                ));
-                controllerContainer.controller = controller;
-                this._workerTokenizerControllers.set(controller.controllerId, controller);
-                return toDisposable(() => {
-                    controllerContainer.controller = undefined;
-                    this._workerTokenizerControllers.delete(controller.controllerId);
-                    controller.dispose();
-                });
-            }));
-            return controllerContainer;
-        });
-        return {
-            dispose() {
-                store.dispose();
-            },
-            requestTokens: async (startLineNumber, endLineNumberExclusive) => {
-                const container = await controllerContainer;
-                if (container?.controller && container.worker === this._worker) {
-                    container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
-                }
-            },
-            reportMismatchingTokens: (lineNumber) => {
-                if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
-                    return;
-                }
-                ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
-                this._notificationService.error({
-                    message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
-                    name: 'Async Tokenization Token Mismatch',
-                });
-                this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
-            },
-        };
-    }
-    setGrammarDefinitions(grammarDefinitions) {
-        this._grammarDefinitions = grammarDefinitions;
-        this._disposeWorker();
-    }
-    acceptTheme(theme, colorMap) {
-        this._currentTheme = theme;
-        this._currentTokenColorMap = colorMap;
-        if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
-            this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
-        }
-    }
-    _getWorkerProxy() {
-        if (!this._workerProxyPromise) {
-            this._workerProxyPromise = this._createWorkerProxy();
-        }
-        return this._workerProxyPromise;
-    }
-    async _createWorkerProxy() {
-        const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
-        const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
-        const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
-        const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
-        const useAsar = this._environmentService.isBuilt && !isWeb;
-        const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
-        const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
-        const textmateMain = `${textmateLocation}/release/main.js`;
-        const onigurumaMain = `${onigurumaLocation}/release/main.js`;
-        const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
-        const uri = ( ( FileAccess.asBrowserUri(textmateMain)).toString(true));
-        const createData = {
-            grammarDefinitions: this._grammarDefinitions,
-            textmateMainUri: uri,
-            onigurumaMainUri: ( ( FileAccess.asBrowserUri(onigurumaMain)).toString(true)),
-            onigurumaWASMUri: ( ( FileAccess.asBrowserUri(onigurumaWASM)).toString(true)),
-        };
-        const host = {
-            readFile: async (_resource) => {
-                const resource = URI.revive(_resource);
-                return this._extensionResourceLoaderService.readExtensionResource(resource);
-            },
-            setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
-                const controller = this._workerTokenizerControllers.get(controllerId);
-                if (controller) {
-                    controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
-                }
-            },
-            reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
-                this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
-            }
-        };
-        const worker = this._worker = createWebWorker(this._modelService, this._languageConfigurationService, {
-            createData,
-            label: 'textMateWorker',
-            moduleId: 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker',
-            host,
-        });
-        const proxy = await ( worker.getProxy());
-        if (this._worker !== worker) {
-            return null;
-        }
-        this._workerProxy = proxy;
-        if (this._currentTheme && this._currentTokenColorMap) {
-            this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
-        }
-        return proxy;
-    }
-    _disposeWorker() {
-        for (const controller of ( this._workerTokenizerControllers.values())) {
-            controller.dispose();
-        }
-        this._workerTokenizerControllers.clear();
-        if (this._worker) {
-            this._worker.dispose();
-            this._worker = null;
-        }
-        this._workerProxy = null;
-        this._workerProxyPromise = null;
-    }
-};
-ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
-    ( __param(2, IExtensionResourceLoaderService)),
-    ( __param(3, IModelService)),
-    ( __param(4, ILanguageConfigurationService)),
-    ( __param(5, IConfigurationService)),
-    ( __param(6, ILanguageService)),
-    ( __param(7, IEnvironmentService)),
-    ( __param(8, INotificationService)),
-    ( __param(9, ITelemetryService))
-], ThreadedBackgroundTokenizerFactory));
-function keepAliveWhenAttached(textModel, factory) {
-    const disposableStore = ( new DisposableStore());
-    const subStore = disposableStore.add(( new DisposableStore()));
-    function checkAttached() {
-        if (textModel.isAttachedToEditor()) {
-            subStore.add(factory());
-        }
-        else {
-            subStore.clear();
-        }
-    }
-    checkAttached();
-    disposableStore.add(textModel.onDidChangeAttached(() => {
-        checkAttached();
-    }));
-    return disposableStore;
-}
-export { ThreadedBackgroundTokenizerFactory };

package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js

@@ -1,87 +0,0 @@
-import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
-import { TMGrammarFactory } from '../../../common/TMGrammarFactory.js';
-import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js';
-import * as vscodeTextmate from 'vscode-textmate';
-import * as vscodeOniguruma from 'vscode-oniguruma';
-class TextMateTokenizationWorker {
-    constructor(ctx, _createData) {
-        this._createData = _createData;
-        this._models = ( new Map());
-        this._grammarCache = [];
-        this._host = ctx.host;
-        const grammarDefinitions = ( _createData.grammarDefinitions.map((def) => {
-            return {
-                location: URI.revive(def.location),
-                language: def.language,
-                scopeName: def.scopeName,
-                embeddedLanguages: def.embeddedLanguages,
-                tokenTypes: def.tokenTypes,
-                injectTo: def.injectTo,
-                balancedBracketSelectors: def.balancedBracketSelectors,
-                unbalancedBracketSelectors: def.unbalancedBracketSelectors,
-                sourceExtensionId: def.sourceExtensionId,
-            };
-        }));
-        this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
-    }
-    async _loadTMGrammarFactory(grammarDefinitions) {
-        const response = await fetch(this._createData.onigurumaWASMUri);
-        const bytes = await response.arrayBuffer();
-        await vscodeOniguruma.loadWASM(bytes);
-        const onigLib = Promise.resolve({
-            createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
-            createOnigString: (str) => vscodeOniguruma.createOnigString(str)
-        });
-        return ( new TMGrammarFactory({
-            logTrace: (msg) => { },
-            logError: (msg, err) => console.error(msg, err),
-            readFile: (resource) => this._host.readFile(resource)
-        }, grammarDefinitions, vscodeTextmate, onigLib));
-    }
-    acceptNewModel(data) {
-        const uri = URI.revive(data.uri);
-        const that = this;
-        this._models.set(data.controllerId, ( new TextMateWorkerTokenizer(uri, data.lines, data.EOL, data.versionId, {
-            async getOrCreateGrammar(languageId, encodedLanguageId) {
-                const grammarFactory = await that._grammarFactory;
-                if (!grammarFactory) {
-                    return Promise.resolve(null);
-                }
-                if (!that._grammarCache[encodedLanguageId]) {
-                    that._grammarCache[encodedLanguageId] = grammarFactory.createGrammar(languageId, encodedLanguageId);
-                }
-                return that._grammarCache[encodedLanguageId];
-            },
-            setTokensAndStates(versionId, tokens, stateDeltas) {
-                that._host.setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
-            },
-            reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
-                that._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
-            },
-        }, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength)));
-    }
-    acceptModelChanged(controllerId, e) {
-        this._models.get(controllerId).onEvents(e);
-    }
-    retokenize(controllerId, startLineNumber, endLineNumberExclusive) {
-        this._models.get(controllerId).retokenize(startLineNumber, endLineNumberExclusive);
-    }
-    acceptModelLanguageChanged(controllerId, newLanguageId, newEncodedLanguageId) {
-        this._models.get(controllerId).onLanguageId(newLanguageId, newEncodedLanguageId);
-    }
-    acceptRemovedModel(controllerId) {
-        const model = this._models.get(controllerId);
-        if (model) {
-            model.dispose();
-            this._models.delete(controllerId);
-        }
-    }
-    async acceptTheme(theme, colorMap) {
-        const grammarFactory = await this._grammarFactory;
-        grammarFactory?.setTheme(theme, colorMap);
-    }
-    acceptMaxTokenizationLineLength(controllerId, value) {
-        this._models.get(controllerId).acceptMaxTokenizationLineLength(value);
-    }
-}
-export { TextMateTokenizationWorker };

package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js

@@ -1,140 +0,0 @@
-import { RunOnceScheduler } from 'monaco-editor/esm/vs/base/common/async.js';
-import { observableValue } from 'monaco-editor/esm/vs/base/common/observable.js';
-import { setTimeout0 } from 'monaco-editor/esm/vs/base/common/platform.js';
-import { LineRange } from 'monaco-editor/esm/vs/editor/common/core/lineRange.js';
-import { MirrorTextModel } from 'monaco-editor/esm/vs/editor/common/model/mirrorTextModel.js';
-import { TokenizerWithStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
-import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
-import { LineTokens } from 'monaco-editor/esm/vs/editor/common/tokens/lineTokens.js';
-import { TextMateTokenizationSupport } from '../../tokenizationSupport/textMateTokenizationSupport.js';
-import { TokenizationSupportWithLineLimit } from '../../tokenizationSupport/tokenizationSupportWithLineLimit.js';
-import { diffStateStacksRefEq } from 'vscode-textmate';
-class TextMateWorkerTokenizer extends MirrorTextModel {
-    constructor(uri, lines, eol, versionId, _host, _languageId, _encodedLanguageId, maxTokenizationLineLength) {
-        super(uri, lines, eol, versionId);
-        this._host = _host;
-        this._languageId = _languageId;
-        this._encodedLanguageId = _encodedLanguageId;
-        this._tokenizerWithStateStore = null;
-        this._isDisposed = false;
-        this._maxTokenizationLineLength = observableValue('_maxTokenizationLineLength', -1);
-        this._tokenizeDebouncer = ( new RunOnceScheduler(() => this._tokenize(), 10));
-        this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined);
-        this._resetTokenization();
-    }
-    dispose() {
-        this._isDisposed = true;
-        super.dispose();
-    }
-    onLanguageId(languageId, encodedLanguageId) {
-        this._languageId = languageId;
-        this._encodedLanguageId = encodedLanguageId;
-        this._resetTokenization();
-    }
-    onEvents(e) {
-        super.onEvents(e);
-        this._tokenizerWithStateStore?.store.acceptChanges(e.changes);
-        this._tokenizeDebouncer.schedule();
-    }
-    acceptMaxTokenizationLineLength(maxTokenizationLineLength) {
-        this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined);
-    }
-    retokenize(startLineNumber, endLineNumberExclusive) {
-        if (this._tokenizerWithStateStore) {
-            this._tokenizerWithStateStore.store.invalidateEndStateRange(( new LineRange(startLineNumber, endLineNumberExclusive)));
-            this._tokenizeDebouncer.schedule();
-        }
-    }
-    async _resetTokenization() {
-        this._tokenizerWithStateStore = null;
-        const languageId = this._languageId;
-        const encodedLanguageId = this._encodedLanguageId;
-        const r = await this._host.getOrCreateGrammar(languageId, encodedLanguageId);
-        if (this._isDisposed || languageId !== this._languageId || encodedLanguageId !== this._encodedLanguageId || !r) {
-            return;
-        }
-        if (r.grammar) {
-            const tokenizationSupport = ( new TokenizationSupportWithLineLimit(this._encodedLanguageId, ( new TextMateTokenizationSupport(
-                r.grammar,
-                r.initialState,
-                false,
-                undefined,
-                () => false,
-                (timeMs, lineLength, isRandomSample) => {
-                    this._host.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, isRandomSample);
-                },
-                false
-            )), this._maxTokenizationLineLength));
-            this._tokenizerWithStateStore = ( new TokenizerWithStateStore(this._lines.length, tokenizationSupport));
-        }
-        else {
-            this._tokenizerWithStateStore = null;
-        }
-        this._tokenize();
-    }
-    async _tokenize() {
-        if (this._isDisposed || !this._tokenizerWithStateStore) {
-            return;
-        }
-        if (!this._diffStateStacksRefEqFn) {
-            this._diffStateStacksRefEqFn = diffStateStacksRefEq;
-        }
-        const startTime = ( new Date()).getTime();
-        while (true) {
-            let tokenizedLines = 0;
-            const tokenBuilder = ( new ContiguousMultilineTokensBuilder());
-            const stateDeltaBuilder = ( new StateDeltaBuilder());
-            while (true) {
-                const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine();
-                if (lineToTokenize === null || tokenizedLines > 200) {
-                    break;
-                }
-                tokenizedLines++;
-                const text = this._lines[lineToTokenize.lineNumber - 1];
-                const r = this._tokenizerWithStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineToTokenize.startState);
-                if (this._tokenizerWithStateStore.store.setEndState(lineToTokenize.lineNumber, r.endState)) {
-                    const delta = this._diffStateStacksRefEqFn(lineToTokenize.startState, r.endState);
-                    stateDeltaBuilder.setState(lineToTokenize.lineNumber, delta);
-                }
-                else {
-                    stateDeltaBuilder.setState(lineToTokenize.lineNumber, null);
-                }
-                LineTokens.convertToEndOffset(r.tokens, text.length);
-                tokenBuilder.add(lineToTokenize.lineNumber, r.tokens);
-                const deltaMs = ( new Date()).getTime() - startTime;
-                if (deltaMs > 20) {
-                    break;
-                }
-            }
-            if (tokenizedLines === 0) {
-                break;
-            }
-            const stateDeltas = stateDeltaBuilder.getStateDeltas();
-            this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), stateDeltas);
-            const deltaMs = ( new Date()).getTime() - startTime;
-            if (deltaMs > 20) {
-                setTimeout0(() => this._tokenize());
-                return;
-            }
-        }
-    }
-}
-class StateDeltaBuilder {
-    constructor() {
-        this._lastStartLineNumber = -1;
-        this._stateDeltas = [];
-    }
-    setState(lineNumber, stackDiff) {
-        if (lineNumber === this._lastStartLineNumber + 1) {
-            this._stateDeltas[this._stateDeltas.length - 1].stateDeltas.push(stackDiff);
-        }
-        else {
-            this._stateDeltas.push({ startLineNumber: lineNumber, stateDeltas: [stackDiff] });
-        }
-        this._lastStartLineNumber = lineNumber;
-    }
-    getStateDeltas() {
-        return this._stateDeltas;
-    }
-}
-export { TextMateWorkerTokenizer };

package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.d.ts

@@ -1,11 +0,0 @@
-import { ServiceIdentifier } from '../../../../platform/instantiation/common/instantiation.js';
-import { IGrammar } from 'vscode-textmate';
-
-declare const ITextMateTokenizationService: ServiceIdentifier<ITextMateTokenizationService>;
-interface ITextMateTokenizationService {
-    readonly _serviceBrand: undefined;
-    createTokenizer(languageId: string): Promise<IGrammar | null>;
-    startDebugMode(printFn: (str: string) => void, onStop: () => void): void;
-}
-
-export { ITextMateTokenizationService };