@codingame/monaco-vscode-api 17.2.1 → 18.0.0

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.js

@@ -1,507 +0,0 @@
-
-import { __decorate, __param } from '../../../../../../external/tslib/tslib.es6.js';
-import { CharCode } from '../../../base/common/charCode.js';
-import { BugIndicatingError, onUnexpectedError } from '../../../base/common/errors.js';
-import { Emitter, Event } from '../../../base/common/event.js';
-import { DisposableStore, MutableDisposable, DisposableMap } from '../../../base/common/lifecycle.js';
-import { countEOL } from '../core/eolCounter.js';
-import { LineRange } from '../core/lineRange.js';
-import { Position } from '../core/position.js';
-import { getWordAtText } from '../core/wordHelper.js';
-import { StandardTokenType } from '../encodedTokenAttributes.js';
-import { TreeSitterTokenizationRegistry, TokenizationRegistry } from '../languages.js';
-import { ILanguageService } from '../languages/language.service.js';
-import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.service.js';
-import { TextModelPart } from './textModelPart.js';
-import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from './textModelTokens.js';
-import { AbstractTokens, AttachedViewHandler } from './tokens.js';
-import { TreeSitterTokens } from './treeSitterTokens.js';
-import { BackgroundTokenizationState } from '../tokenizationTextModelPart.js';
-import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
-import { ContiguousTokensStore } from '../tokens/contiguousTokensStore.js';
-import { SparseTokensStore } from '../tokens/sparseTokensStore.js';
-import { IInstantiationService } from '../../../platform/instantiation/common/instantiation.js';
-
-var TokenizationTextModelPart_1;
-let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
-    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _instantiationService) {
-        super();
-        this._textModel = _textModel;
-        this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
-        this._languageId = _languageId;
-        this._attachedViews = _attachedViews;
-        this._languageService = _languageService;
-        this._languageConfigurationService = _languageConfigurationService;
-        this._instantiationService = _instantiationService;
-        this._semanticTokens = ( new SparseTokensStore(this._languageService.languageIdCodec));
-        this._onDidChangeLanguage = this._register(( new Emitter()));
-        this.onDidChangeLanguage = this._onDidChangeLanguage.event;
-        this._onDidChangeLanguageConfiguration = this._register(( new Emitter()));
-        this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
-        this._onDidChangeTokens = this._register(( new Emitter()));
-        this.onDidChangeTokens = this._onDidChangeTokens.event;
-        this._tokensDisposables = this._register(( new DisposableStore()));
-        this._register(Event.filter(TreeSitterTokenizationRegistry.onDidChange, (e) => e.changedLanguages.includes(this._languageId))(() => {
-            this.createPreferredTokenProvider();
-        }));
-        this.createPreferredTokenProvider();
-    }
-    createGrammarTokens() {
-        return this._register(( new GrammarTokens(
-            this._languageService.languageIdCodec,
-            this._textModel,
-            () => this._languageId,
-            this._attachedViews
-        )));
-    }
-    createTreeSitterTokens() {
-        return this._register(this._instantiationService.createInstance(TreeSitterTokens, this._languageService.languageIdCodec, this._textModel, () => this._languageId));
-    }
-    createTokens(useTreeSitter) {
-        const needsReset = this._tokens !== undefined;
-        this._tokens?.dispose();
-        this._tokens = useTreeSitter ? this.createTreeSitterTokens() : this.createGrammarTokens();
-        this._tokensDisposables.clear();
-        this._tokensDisposables.add(this._tokens.onDidChangeTokens(e => {
-            this._emitModelTokensChangedEvent(e);
-        }));
-        this._tokensDisposables.add(this._tokens.onDidChangeBackgroundTokenizationState(e => {
-            this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
-        }));
-        if (needsReset) {
-            this._tokens.resetTokenization();
-        }
-    }
-    createPreferredTokenProvider() {
-        if (TreeSitterTokenizationRegistry.get(this._languageId)) {
-            if (!(this._tokens instanceof TreeSitterTokens)) {
-                this.createTokens(true);
-            }
-        }
-        else {
-            if (!(this._tokens instanceof GrammarTokens)) {
-                this.createTokens(false);
-            }
-        }
-    }
-    _hasListeners() {
-        return (this._onDidChangeLanguage.hasListeners()
-            || this._onDidChangeLanguageConfiguration.hasListeners()
-            || this._onDidChangeTokens.hasListeners());
-    }
-    handleLanguageConfigurationServiceChange(e) {
-        if (e.affects(this._languageId)) {
-            this._onDidChangeLanguageConfiguration.fire({});
-        }
-    }
-    handleDidChangeContent(e) {
-        if (e.isFlush) {
-            this._semanticTokens.flush();
-        }
-        else if (!e.isEolChange) {
-            for (const c of e.changes) {
-                const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
-                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null);
-            }
-        }
-        this._tokens.handleDidChangeContent(e);
-    }
-    handleDidChangeAttached() {
-        this._tokens.handleDidChangeAttached();
-    }
-    getLineTokens(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        const syntacticTokens = this._tokens.getLineTokens(lineNumber);
-        return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
-    }
-    _emitModelTokensChangedEvent(e) {
-        if (!this._textModel._isDisposing()) {
-            this._bracketPairsTextModelPart.handleDidChangeTokens(e);
-            this._onDidChangeTokens.fire(e);
-        }
-    }
-    validateLineNumber(lineNumber) {
-        if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
-            throw ( new BugIndicatingError('Illegal value for lineNumber'));
-        }
-    }
-    get hasTokens() {
-        return this._tokens.hasTokens;
-    }
-    resetTokenization() {
-        this._tokens.resetTokenization();
-    }
-    get backgroundTokenizationState() {
-        return this._tokens.backgroundTokenizationState;
-    }
-    forceTokenization(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        this._tokens.forceTokenization(lineNumber);
-    }
-    hasAccurateTokensForLine(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        return this._tokens.hasAccurateTokensForLine(lineNumber);
-    }
-    isCheapToTokenize(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        return this._tokens.isCheapToTokenize(lineNumber);
-    }
-    tokenizeIfCheap(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        this._tokens.tokenizeIfCheap(lineNumber);
-    }
-    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
-        return this._tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
-    }
-    tokenizeLinesAt(lineNumber, lines) {
-        return this._tokens.tokenizeLinesAt(lineNumber, lines);
-    }
-    setSemanticTokens(tokens, isComplete) {
-        this._semanticTokens.set(tokens, isComplete, this._textModel);
-        this._emitModelTokensChangedEvent({
-            semanticTokensApplied: tokens !== null,
-            ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],
-        });
-    }
-    hasCompleteSemanticTokens() {
-        return this._semanticTokens.isComplete();
-    }
-    hasSomeSemanticTokens() {
-        return !this._semanticTokens.isEmpty();
-    }
-    setPartialSemanticTokens(range, tokens) {
-        if (this.hasCompleteSemanticTokens()) {
-            return;
-        }
-        const changedRange = this._textModel.validateRange(this._semanticTokens.setPartial(range, tokens));
-        this._emitModelTokensChangedEvent({
-            semanticTokensApplied: true,
-            ranges: [
-                {
-                    fromLineNumber: changedRange.startLineNumber,
-                    toLineNumber: changedRange.endLineNumber,
-                },
-            ],
-        });
-    }
-    getWordAtPosition(_position) {
-        this.assertNotDisposed();
-        const position = this._textModel.validatePosition(_position);
-        const lineContent = this._textModel.getLineContent(position.lineNumber);
-        const lineTokens = this.getLineTokens(position.lineNumber);
-        const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
-        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex);
-        const rightBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), lineContent.substring(rbStartOffset, rbEndOffset), rbStartOffset);
-        if (rightBiasedWord &&
-            rightBiasedWord.startColumn <= _position.column &&
-            _position.column <= rightBiasedWord.endColumn) {
-            return rightBiasedWord;
-        }
-        if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
-            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex - 1);
-            const leftBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), lineContent.substring(lbStartOffset, lbEndOffset), lbStartOffset);
-            if (leftBiasedWord &&
-                leftBiasedWord.startColumn <= _position.column &&
-                _position.column <= leftBiasedWord.endColumn) {
-                return leftBiasedWord;
-            }
-        }
-        return null;
-    }
-    getLanguageConfiguration(languageId) {
-        return this._languageConfigurationService.getLanguageConfiguration(languageId);
-    }
-    static _findLanguageBoundaries(lineTokens, tokenIndex) {
-        const languageId = lineTokens.getLanguageId(tokenIndex);
-        let startOffset = 0;
-        for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {
-            startOffset = lineTokens.getStartOffset(i);
-        }
-        let endOffset = lineTokens.getLineContent().length;
-        for (let i = tokenIndex, tokenCount = lineTokens.getCount(); i < tokenCount && lineTokens.getLanguageId(i) === languageId; i++) {
-            endOffset = lineTokens.getEndOffset(i);
-        }
-        return [startOffset, endOffset];
-    }
-    getWordUntilPosition(position) {
-        const wordAtPosition = this.getWordAtPosition(position);
-        if (!wordAtPosition) {
-            return { word: '', startColumn: position.column, endColumn: position.column, };
-        }
-        return {
-            word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
-            startColumn: wordAtPosition.startColumn,
-            endColumn: position.column,
-        };
-    }
-    getLanguageId() {
-        return this._languageId;
-    }
-    getLanguageIdAtPosition(lineNumber, column) {
-        const position = this._textModel.validatePosition(( new Position(lineNumber, column)));
-        const lineTokens = this.getLineTokens(position.lineNumber);
-        return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
-    }
-    setLanguageId(languageId, source = 'api') {
-        if (this._languageId === languageId) {
-            return;
-        }
-        const e = {
-            oldLanguage: this._languageId,
-            newLanguage: languageId,
-            source
-        };
-        this._languageId = languageId;
-        this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
-        this._tokens.resetTokenization();
-        this.createPreferredTokenProvider();
-        this._onDidChangeLanguage.fire(e);
-        this._onDidChangeLanguageConfiguration.fire({});
-    }
-};
-TokenizationTextModelPart = TokenizationTextModelPart_1 = ( __decorate([
-    ( __param(4, ILanguageService)),
-    ( __param(5, ILanguageConfigurationService)),
-    ( __param(6, IInstantiationService))
-], TokenizationTextModelPart));
-class GrammarTokens extends AbstractTokens {
-    constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
-        super(languageIdCodec, textModel, getLanguageId);
-        this._tokenizer = null;
-        this._backgroundTokenizationState = BackgroundTokenizationState.InProgress;
-        this._onDidChangeBackgroundTokenizationState = this._register(( new Emitter()));
-        this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
-        this._defaultBackgroundTokenizer = null;
-        this._backgroundTokenizer = this._register(( new MutableDisposable()));
-        this._tokens = ( new ContiguousTokensStore(this._languageIdCodec));
-        this._debugBackgroundTokenizer = this._register(( new MutableDisposable()));
-        this._attachedViewStates = this._register(( new DisposableMap()));
-        this._register(TokenizationRegistry.onDidChange((e) => {
-            const languageId = this.getLanguageId();
-            if (e.changedLanguages.indexOf(languageId) === -1) {
-                return;
-            }
-            this.resetTokenization();
-        }));
-        this.resetTokenization();
-        this._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {
-            if (state) {
-                let existing = this._attachedViewStates.get(view);
-                if (!existing) {
-                    existing = ( new AttachedViewHandler(() => this.refreshRanges(existing.lineRanges)));
-                    this._attachedViewStates.set(view, existing);
-                }
-                existing.handleStateChange(state);
-            }
-            else {
-                this._attachedViewStates.deleteAndDispose(view);
-            }
-        }));
-    }
-    resetTokenization(fireTokenChangeEvent = true) {
-        this._tokens.flush();
-        this._debugBackgroundTokens?.flush();
-        if (this._debugBackgroundStates) {
-            this._debugBackgroundStates = ( new TrackingTokenizationStateStore(this._textModel.getLineCount()));
-        }
-        if (fireTokenChangeEvent) {
-            this._onDidChangeTokens.fire({
-                semanticTokensApplied: false,
-                ranges: [
-                    {
-                        fromLineNumber: 1,
-                        toLineNumber: this._textModel.getLineCount(),
-                    },
-                ],
-            });
-        }
-        const initializeTokenization = () => {
-            if (this._textModel.isTooLargeForTokenization()) {
-                return [null, null];
-            }
-            const tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());
-            if (!tokenizationSupport) {
-                return [null, null];
-            }
-            let initialState;
-            try {
-                initialState = tokenizationSupport.getInitialState();
-            }
-            catch (e) {
-                onUnexpectedError(e);
-                return [null, null];
-            }
-            return [tokenizationSupport, initialState];
-        };
-        const [tokenizationSupport, initialState] = initializeTokenization();
-        if (tokenizationSupport && initialState) {
-            this._tokenizer = ( new TokenizerWithStateStoreAndTextModel(
-                this._textModel.getLineCount(),
-                tokenizationSupport,
-                this._textModel,
-                this._languageIdCodec
-            ));
-        }
-        else {
-            this._tokenizer = null;
-        }
-        this._backgroundTokenizer.clear();
-        this._defaultBackgroundTokenizer = null;
-        if (this._tokenizer) {
-            const b = {
-                setTokens: (tokens) => {
-                    this.setTokens(tokens);
-                },
-                backgroundTokenizationFinished: () => {
-                    if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
-                        return;
-                    }
-                    const newState = BackgroundTokenizationState.Completed;
-                    this._backgroundTokenizationState = newState;
-                    this._onDidChangeBackgroundTokenizationState.fire();
-                },
-                setEndState: (lineNumber, state) => {
-                    if (!this._tokenizer) {
-                        return;
-                    }
-                    const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
-                    if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
-                        this._tokenizer?.store.setEndState(lineNumber, state);
-                    }
-                },
-            };
-            if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
-                this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
-            }
-            if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {
-                this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
-                    ( new DefaultBackgroundTokenizer(this._tokenizer, b));
-                this._defaultBackgroundTokenizer.handleChanges();
-            }
-            if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
-                this._debugBackgroundTokens = ( new ContiguousTokensStore(this._languageIdCodec));
-                this._debugBackgroundStates = ( new TrackingTokenizationStateStore(this._textModel.getLineCount()));
-                this._debugBackgroundTokenizer.clear();
-                this._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {
-                    setTokens: (tokens) => {
-                        this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
-                    },
-                    backgroundTokenizationFinished() {
-                    },
-                    setEndState: (lineNumber, state) => {
-                        this._debugBackgroundStates?.setEndState(lineNumber, state);
-                    },
-                });
-            }
-            else {
-                this._debugBackgroundTokens = undefined;
-                this._debugBackgroundStates = undefined;
-                this._debugBackgroundTokenizer.value = undefined;
-            }
-        }
-        this.refreshAllVisibleLineTokens();
-    }
-    handleDidChangeAttached() {
-        this._defaultBackgroundTokenizer?.handleChanges();
-    }
-    handleDidChangeContent(e) {
-        if (e.isFlush) {
-            this.resetTokenization(false);
-        }
-        else if (!e.isEolChange) {
-            for (const c of e.changes) {
-                const [eolCount, firstLineLength] = countEOL(c.text);
-                this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
-                this._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);
-            }
-            this._debugBackgroundStates?.acceptChanges(e.changes);
-            if (this._tokenizer) {
-                this._tokenizer.store.acceptChanges(e.changes);
-            }
-            this._defaultBackgroundTokenizer?.handleChanges();
-        }
-    }
-    setTokens(tokens) {
-        const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
-        if (changes.length > 0) {
-            this._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });
-        }
-        return { changes: changes };
-    }
-    refreshAllVisibleLineTokens() {
-        const ranges = LineRange.joinMany(( [...this._attachedViewStates].map(([_, s]) => s.lineRanges)));
-        this.refreshRanges(ranges);
-    }
-    refreshRanges(ranges) {
-        for (const range of ranges) {
-            this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
-        }
-    }
-    refreshRange(startLineNumber, endLineNumber) {
-        if (!this._tokenizer) {
-            return;
-        }
-        startLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));
-        endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
-        const builder = ( new ContiguousMultilineTokensBuilder());
-        const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
-        const changedTokens = this.setTokens(builder.finalize());
-        if (heuristicTokens) {
-            for (const c of changedTokens.changes) {
-                this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
-            }
-        }
-        this._defaultBackgroundTokenizer?.checkFinished();
-    }
-    forceTokenization(lineNumber) {
-        const builder = ( new ContiguousMultilineTokensBuilder());
-        this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
-        this.setTokens(builder.finalize());
-        this._defaultBackgroundTokenizer?.checkFinished();
-    }
-    hasAccurateTokensForLine(lineNumber) {
-        if (!this._tokenizer) {
-            return true;
-        }
-        return this._tokenizer.hasAccurateTokensForLine(lineNumber);
-    }
-    isCheapToTokenize(lineNumber) {
-        if (!this._tokenizer) {
-            return true;
-        }
-        return this._tokenizer.isCheapToTokenize(lineNumber);
-    }
-    getLineTokens(lineNumber) {
-        const lineText = this._textModel.getLineContent(lineNumber);
-        const result = this._tokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
-        if (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {
-            if (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {
-                const backgroundResult = this._debugBackgroundTokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
-                if (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {
-                    this._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);
-                }
-            }
-        }
-        return result;
-    }
-    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
-        if (!this._tokenizer) {
-            return StandardTokenType.Other;
-        }
-        const position = this._textModel.validatePosition(( new Position(lineNumber, column)));
-        this.forceTokenization(position.lineNumber);
-        return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
-    }
-    tokenizeLinesAt(lineNumber, lines) {
-        if (!this._tokenizer) {
-            return null;
-        }
-        this.forceTokenization(lineNumber);
-        return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
-    }
-    get hasTokens() {
-        return this._tokens.hasTokens;
-    }
-}
-
-export { TokenizationTextModelPart };

package/vscode/src/vs/editor/common/model/tokens.js

@@ -1,81 +0,0 @@
-
-import { equals } from '../../../base/common/arrays.js';
-import { RunOnceScheduler } from '../../../base/common/async.js';
-import { Emitter } from '../../../base/common/event.js';
-import { Disposable } from '../../../base/common/lifecycle.js';
-import { LineRange } from '../core/lineRange.js';
-
-class AttachedViews {
-    constructor() {
-        this._onDidChangeVisibleRanges = ( new Emitter());
-        this.onDidChangeVisibleRanges = this._onDidChangeVisibleRanges.event;
-        this._views = ( new Set());
-    }
-    attachView() {
-        const view = ( new AttachedViewImpl((state) => {
-            this._onDidChangeVisibleRanges.fire({ view, state });
-        }));
-        this._views.add(view);
-        return view;
-    }
-    detachView(view) {
-        this._views.delete(view);
-        this._onDidChangeVisibleRanges.fire({ view, state: undefined });
-    }
-}
-class AttachedViewImpl {
-    constructor(handleStateChange) {
-        this.handleStateChange = handleStateChange;
-    }
-    setVisibleLines(visibleLines, stabilized) {
-        const visibleLineRanges = ( visibleLines.map((line) => ( new LineRange(line.startLineNumber, line.endLineNumber + 1))));
-        this.handleStateChange({ visibleLineRanges, stabilized });
-    }
-}
-class AttachedViewHandler extends Disposable {
-    get lineRanges() { return this._lineRanges; }
-    constructor(_refreshTokens) {
-        super();
-        this._refreshTokens = _refreshTokens;
-        this.runner = this._register(( new RunOnceScheduler(() => this.update(), 50)));
-        this._computedLineRanges = [];
-        this._lineRanges = [];
-    }
-    update() {
-        if (equals(this._computedLineRanges, this._lineRanges, (a, b) => a.equals(b))) {
-            return;
-        }
-        this._computedLineRanges = this._lineRanges;
-        this._refreshTokens();
-    }
-    handleStateChange(state) {
-        this._lineRanges = state.visibleLineRanges;
-        if (state.stabilized) {
-            this.runner.cancel();
-            this.update();
-        }
-        else {
-            this.runner.schedule();
-        }
-    }
-}
-class AbstractTokens extends Disposable {
-    get backgroundTokenizationState() {
-        return this._backgroundTokenizationState;
-    }
-    constructor(_languageIdCodec, _textModel, getLanguageId) {
-        super();
-        this._languageIdCodec = _languageIdCodec;
-        this._textModel = _textModel;
-        this.getLanguageId = getLanguageId;
-        this._onDidChangeTokens = this._register(( new Emitter()));
-        this.onDidChangeTokens = this._onDidChangeTokens.event;
-    }
-    tokenizeIfCheap(lineNumber) {
-        if (this.isCheapToTokenize(lineNumber)) {
-            this.forceTokenization(lineNumber);
-        }
-    }
-}
-
-export { AbstractTokens, AttachedViewHandler, AttachedViews };

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.service.d.ts

@@ -1,24 +0,0 @@
-import { Range } from "../core/range.js";
-import { ITextModel } from "../model.js";
-import { IModelContentChangedEvent } from "../textModelEvents.js";
-import { TokenUpdate, TokenQuality } from "@codingame/monaco-vscode-treesitter-service-override/vscode/vs/editor/common/model/tokenStore";
-export interface ITreeSitterTokenizationStoreService {
-    readonly _serviceBrand: undefined;
-    setTokens(model: ITextModel, tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
-    handleContentChanged(model: ITextModel, e: IModelContentChangedEvent): void;
-    getTokens(model: ITextModel, line: number): Uint32Array | undefined;
-    updateTokens(model: ITextModel, version: number, updates: {
-        oldRangeLength?: number;
-        newTokens: TokenUpdate[];
-    }[], tokenQuality: TokenQuality): void;
-    markForRefresh(model: ITextModel, range: Range): void;
-    getNeedsRefresh(model: ITextModel): {
-        range: Range;
-        startOffset: number;
-        endOffset: number;
-    }[];
-    hasTokens(model: ITextModel, accurateForRange?: Range): boolean;
-    rangeHasTokens(model: ITextModel, range: Range, minimumTokenQuality: TokenQuality): boolean;
-    delete(model: ITextModel): void;
-}
-export declare const ITreeSitterTokenizationStoreService: import("../../../platform/instantiation/common/instantiation.js").ServiceIdentifier<ITreeSitterTokenizationStoreService>;

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.service.js

@@ -1,6 +0,0 @@
-
-import { createDecorator } from '../../../platform/instantiation/common/instantiation.js';
-
-const ITreeSitterTokenizationStoreService = ( createDecorator('treeSitterTokenizationStoreService'));
-
-export { ITreeSitterTokenizationStoreService };

package/vscode/src/vs/editor/common/model/treeSitterTokens.d.ts

@@ -1,31 +0,0 @@
-import { ILanguageIdCodec } from "../languages.js";
-import { LineTokens } from "../tokens/lineTokens.js";
-import { StandardTokenType } from "../encodedTokenAttributes.js";
-import { TextModel } from "./textModel.js";
-import { IModelContentChangedEvent } from "../textModelEvents.js";
-import { AbstractTokens } from "./tokens.js";
-import { ITreeSitterTokenizationStoreService } from "./treeSitterTokenStoreService.service.js";
-import { BackgroundTokenizationState } from "../tokenizationTextModelPart.js";
-import { Emitter, Event } from "../../../base/common/event.js";
-export declare class TreeSitterTokens extends AbstractTokens {
-    private readonly _tokenStore;
-    private _tokenizationSupport;
-    protected _backgroundTokenizationState: BackgroundTokenizationState;
-    protected readonly _onDidChangeBackgroundTokenizationState: Emitter<void>;
-    readonly onDidChangeBackgroundTokenizationState: Event<void>;
-    private _lastLanguageId;
-    private readonly _tokensChangedListener;
-    private readonly _onDidChangeBackgroundTokenization;
-    constructor(languageIdCodec: ILanguageIdCodec, textModel: TextModel, languageId: () => string, _tokenStore: ITreeSitterTokenizationStoreService);
-    private _initialize;
-    getLineTokens(lineNumber: number): LineTokens;
-    resetTokenization(fireTokenChangeEvent?: boolean): void;
-    handleDidChangeAttached(): void;
-    handleDidChangeContent(e: IModelContentChangedEvent): void;
-    forceTokenization(lineNumber: number): void;
-    hasAccurateTokensForLine(lineNumber: number): boolean;
-    isCheapToTokenize(lineNumber: number): boolean;
-    getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
-    tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
-    get hasTokens(): boolean;
-}