@codingame/monaco-vscode-api 8.0.3 → 9.0.0

package/vscode/src/vs/editor/common/model/textModel.js

@@ -10,7 +10,6 @@ import { Constants } from '../../../base/common/uint.js';
 import { URI } from '../../../base/common/uri.js';
 import { countEOL } from '../core/eolCounter.js';
 import { normalizeIndentation } from '../core/indentation.js';
-import { LineRange } from '../core/lineRange.js';
 import { Position } from '../core/position.js';
 import { Range } from '../core/range.js';
 import { Selection } from '../core/selection.js';
@@ -28,7 +27,9 @@ import { PieceTreeTextBuffer } from './pieceTreeTextBuffer/pieceTreeTextBuffer.j
 import { PieceTreeTextBufferBuilder } from './pieceTreeTextBuffer/pieceTreeTextBufferBuilder.js';
 import { TextModelSearch, SearchParams } from './textModelSearch.js';
 import { TokenizationTextModelPart } from './tokenizationTextModelPart.js';
+import { AttachedViews } from './tokens.js';
 import { InternalModelContentChangeEvent, ModelRawContentChangedEvent, ModelRawFlush, ModelRawEOLChanged, LineInjectedText, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted, ModelInjectedTextChangedEvent } from '../textModelEvents.js';
+import { IInstantiationService } from '../../../platform/instantiation/common/instantiation.js';
 import { IUndoRedoService } from '../../../platform/undoRedo/common/undoRedo.service.js';
 var TextModel_1;
 function createTextBufferFactory(text) {
@@ -166,11 +167,12 @@ let TextModel = class TextModel extends Disposable {
     get tokenization() { return this._tokenizationTextModelPart; }
     get bracketPairs() { return this._bracketPairs; }
     get guides() { return this._guidesTextModelPart; }
-    constructor(source, languageIdOrSelection, creationOptions, associatedResource = null, _undoRedoService, _languageService, _languageConfigurationService) {
+    constructor(source, languageIdOrSelection, creationOptions, associatedResource = null, _undoRedoService, _languageService, _languageConfigurationService, instantiationService) {
         super();
         this._undoRedoService = _undoRedoService;
         this._languageService = _languageService;
         this._languageConfigurationService = _languageConfigurationService;
+        this.instantiationService = instantiationService;
         this._onWillDispose = this._register(( new Emitter()));
         this.onWillDispose = this._onWillDispose.event;
         this._onDidChangeDecorations = this._register(( new DidChangeDecorationsEmitter(
@@ -207,14 +209,7 @@ let TextModel = class TextModel extends Disposable {
         this._bracketPairs = this._register(( new BracketPairsTextModelPart(this, this._languageConfigurationService)));
         this._guidesTextModelPart = this._register(( new GuidesTextModelPart(this, this._languageConfigurationService)));
         this._decorationProvider = this._register(( new ColorizedBracketPairsDecorationProvider(this)));
-        this._tokenizationTextModelPart = ( new TokenizationTextModelPart(
-            this._languageService,
-            this._languageConfigurationService,
-            this,
-            this._bracketPairs,
-            languageId,
-            this._attachedViews
-        ));
+        this._tokenizationTextModelPart = this.instantiationService.createInstance(TokenizationTextModelPart, this, this._bracketPairs, languageId, this._attachedViews);
         const bufferLineCount = this._buffer.getLineCount();
         const bufferTextLength = this._buffer.getValueLengthInRange(( new Range(1, 1, bufferLineCount, this._buffer.getLineLength(bufferLineCount) + 1)), EndOfLinePreference.TextDefined);
         if (creationOptions.largeFileOptimizations) {
@@ -1498,7 +1493,8 @@ let TextModel = class TextModel extends Disposable {
 TextModel = TextModel_1 = ( __decorate([
     ( __param(4, IUndoRedoService)),
     ( __param(5, ILanguageService)),
-    ( __param(6, ILanguageConfigurationService))
+    ( __param(6, ILanguageConfigurationService)),
+    ( __param(7, IInstantiationService))
 ], TextModel));
 function indentOfLine(line) {
     let indent = 0;
@@ -1895,31 +1891,4 @@ class DidChangeContentEmitter extends Disposable {
         this._slowEmitter.fire(e);
     }
 }
-class AttachedViews {
-    constructor() {
-        this._onDidChangeVisibleRanges = ( new Emitter());
-        this.onDidChangeVisibleRanges = this._onDidChangeVisibleRanges.event;
-        this._views = ( new Set());
-    }
-    attachView() {
-        const view = ( new AttachedViewImpl((state) => {
-            this._onDidChangeVisibleRanges.fire({ view, state });
-        }));
-        this._views.add(view);
-        return view;
-    }
-    detachView(view) {
-        this._views.delete(view);
-        this._onDidChangeVisibleRanges.fire({ view, state: undefined });
-    }
-}
-class AttachedViewImpl {
-    constructor(handleStateChange) {
-        this.handleStateChange = handleStateChange;
-    }
-    setVisibleLines(visibleLines, stabilized) {
-        const visibleLineRanges = ( visibleLines.map((line) => ( new LineRange(line.startLineNumber, line.endLineNumber + 1))));
-        this.handleStateChange({ visibleLineRanges, stabilized });
-    }
-}
-export { AttachedViews, ModelDecorationGlyphMarginOptions, ModelDecorationInjectedTextOptions, ModelDecorationMinimapOptions, ModelDecorationOptions, ModelDecorationOverviewRulerOptions, TextModel, createTextBuffer, createTextBufferFactory, createTextBufferFactoryFromSnapshot, createTextBufferFactoryFromStream };
+export { ModelDecorationGlyphMarginOptions, ModelDecorationInjectedTextOptions, ModelDecorationMinimapOptions, ModelDecorationOptions, ModelDecorationOverviewRulerOptions, TextModel, createTextBuffer, createTextBufferFactory, createTextBufferFactoryFromSnapshot, createTextBufferFactoryFromStream, indentOfLine };

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.js

@@ -1,30 +1,36 @@
-import { equals } from '../../../base/common/arrays.js';
-import { RunOnceScheduler } from '../../../base/common/async.js';
+import { __decorate, __param } from '../../../../../../external/tslib/tslib.es6.js';
 import { CharCode } from '../../../base/common/charCode.js';
 import { BugIndicatingError, onUnexpectedError } from '../../../base/common/errors.js';
-import { Emitter } from '../../../base/common/event.js';
-import { Disposable, MutableDisposable, DisposableMap } from '../../../base/common/lifecycle.js';
+import { Emitter, Event } from '../../../base/common/event.js';
+import { DisposableStore, MutableDisposable, DisposableMap } from '../../../base/common/lifecycle.js';
 import { countEOL } from '../core/eolCounter.js';
 import { LineRange } from '../core/lineRange.js';
 import { Position } from '../core/position.js';
 import { getWordAtText } from '../core/wordHelper.js';
 import { StandardTokenType } from '../encodedTokenAttributes.js';
-import { TokenizationRegistry } from '../languages.js';
+import { TreeSitterTokenizationRegistry, TokenizationRegistry } from '../languages.js';
+import { ILanguageService } from '../languages/language.js';
+import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.js';
 import { TextModelPart } from './textModelPart.js';
 import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from './textModelTokens.js';
+import { AbstractTokens, AttachedViewHandler } from './tokens.js';
+import { TreeSitterTokens } from './treeSitterTokens.js';
+import { ITreeSitterParserService } from '../services/treeSitterParserService.js';
 import { BackgroundTokenizationState } from '../tokenizationTextModelPart.js';
 import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
 import { ContiguousTokensStore } from '../tokens/contiguousTokensStore.js';
 import { SparseTokensStore } from '../tokens/sparseTokensStore.js';
-class TokenizationTextModelPart extends TextModelPart {
-    constructor(_languageService, _languageConfigurationService, _textModel, _bracketPairsTextModelPart, _languageId, _attachedViews) {
+var TokenizationTextModelPart_1;
+let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
+    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _treeSitterService) {
         super();
-        this._languageService = _languageService;
-        this._languageConfigurationService = _languageConfigurationService;
         this._textModel = _textModel;
         this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
         this._languageId = _languageId;
         this._attachedViews = _attachedViews;
+        this._languageService = _languageService;
+        this._languageConfigurationService = _languageConfigurationService;
+        this._treeSitterService = _treeSitterService;
         this._semanticTokens = ( new SparseTokensStore(this._languageService.languageIdCodec));
         this._onDidChangeLanguage = this._register(( new Emitter()));
         this.onDidChangeLanguage = this._onDidChangeLanguage.event;
@@ -32,18 +38,59 @@ class TokenizationTextModelPart extends TextModelPart {
         this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
         this._onDidChangeTokens = this._register(( new Emitter()));
         this.onDidChangeTokens = this._onDidChangeTokens.event;
-        this.grammarTokens = this._register(( new GrammarTokens(
+        this._tokensDisposables = this._register(( new DisposableStore()));
+        this._register(this._languageConfigurationService.onDidChange(e => {
+            if (e.affects(this._languageId)) {
+                this._onDidChangeLanguageConfiguration.fire({});
+            }
+        }));
+        this._register(Event.filter(TreeSitterTokenizationRegistry.onDidChange, (e) => e.changedLanguages.includes(this._languageId))(() => {
+            this.createPreferredTokenProvider();
+        }));
+        this.createPreferredTokenProvider();
+    }
+    createGrammarTokens() {
+        return this._register(( new GrammarTokens(
             this._languageService.languageIdCodec,
             this._textModel,
             () => this._languageId,
             this._attachedViews
         )));
-        this._register(this.grammarTokens.onDidChangeTokens(e => {
+    }
+    createTreeSitterTokens() {
+        return this._register(( new TreeSitterTokens(
+            this._treeSitterService,
+            this._languageService.languageIdCodec,
+            this._textModel,
+            () => this._languageId
+        )));
+    }
+    createTokens(useTreeSitter) {
+        const needsReset = this._tokens !== undefined;
+        this._tokens?.dispose();
+        this._tokens = useTreeSitter ? this.createTreeSitterTokens() : this.createGrammarTokens();
+        this._tokensDisposables.clear();
+        this._tokensDisposables.add(this._tokens.onDidChangeTokens(e => {
             this._emitModelTokensChangedEvent(e);
         }));
-        this._register(this.grammarTokens.onDidChangeBackgroundTokenizationState(e => {
+        this._tokensDisposables.add(this._tokens.onDidChangeBackgroundTokenizationState(e => {
             this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
         }));
+        if (needsReset) {
+            this._tokens.resetTokenization();
+        }
+    }
+    createPreferredTokenProvider() {
+        if (TreeSitterTokenizationRegistry.get(this._languageId)) {
+            if (!(this._tokens instanceof TreeSitterTokens)) {
+                this.createTokens(true);
+            }
+        }
+        else {
+            if (!(this._tokens instanceof GrammarTokens)) {
+                this.createTokens(false);
+            }
+        }
     }
     _hasListeners() {
         return (this._onDidChangeLanguage.hasListeners()
@@ -65,14 +112,14 @@ class TokenizationTextModelPart extends TextModelPart {
                 this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null);
             }
         }
-        this.grammarTokens.handleDidChangeContent(e);
+        this._tokens.handleDidChangeContent(e);
     }
     handleDidChangeAttached() {
-        this.grammarTokens.handleDidChangeAttached();
+        this._tokens.handleDidChangeAttached();
     }
     getLineTokens(lineNumber) {
         this.validateLineNumber(lineNumber);
-        const syntacticTokens = this.grammarTokens.getLineTokens(lineNumber);
+        const syntacticTokens = this._tokens.getLineTokens(lineNumber);
         return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
     }
     _emitModelTokensChangedEvent(e) {
@@ -87,35 +134,35 @@ class TokenizationTextModelPart extends TextModelPart {
         }
     }
     get hasTokens() {
-        return this.grammarTokens.hasTokens;
+        return this._tokens.hasTokens;
     }
     resetTokenization() {
-        this.grammarTokens.resetTokenization();
+        this._tokens.resetTokenization();
     }
     get backgroundTokenizationState() {
-        return this.grammarTokens.backgroundTokenizationState;
+        return this._tokens.backgroundTokenizationState;
     }
     forceTokenization(lineNumber) {
         this.validateLineNumber(lineNumber);
-        this.grammarTokens.forceTokenization(lineNumber);
+        this._tokens.forceTokenization(lineNumber);
     }
     hasAccurateTokensForLine(lineNumber) {
         this.validateLineNumber(lineNumber);
-        return this.grammarTokens.hasAccurateTokensForLine(lineNumber);
+        return this._tokens.hasAccurateTokensForLine(lineNumber);
     }
     isCheapToTokenize(lineNumber) {
         this.validateLineNumber(lineNumber);
-        return this.grammarTokens.isCheapToTokenize(lineNumber);
+        return this._tokens.isCheapToTokenize(lineNumber);
     }
     tokenizeIfCheap(lineNumber) {
         this.validateLineNumber(lineNumber);
-        this.grammarTokens.tokenizeIfCheap(lineNumber);
+        this._tokens.tokenizeIfCheap(lineNumber);
     }
     getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
-        return this.grammarTokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
+        return this._tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
     }
     tokenizeLineWithEdit(position, length, newText) {
-        return this.grammarTokens.tokenizeLineWithEdit(position, length, newText);
+        return this._tokens.tokenizeLineWithEdit(position, length, newText);
     }
     setSemanticTokens(tokens, isComplete) {
         this._semanticTokens.set(tokens, isComplete);
@@ -151,7 +198,7 @@ class TokenizationTextModelPart extends TextModelPart {
         const lineContent = this._textModel.getLineContent(position.lineNumber);
         const lineTokens = this.getLineTokens(position.lineNumber);
         const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
-        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);
+        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex);
         const rightBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), lineContent.substring(rbStartOffset, rbEndOffset), rbStartOffset);
         if (rightBiasedWord &&
             rightBiasedWord.startColumn <= _position.column &&
@@ -159,7 +206,7 @@ class TokenizationTextModelPart extends TextModelPart {
             return rightBiasedWord;
         }
         if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
-            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex - 1);
+            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex - 1);
             const leftBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), lineContent.substring(lbStartOffset, lbEndOffset), lbStartOffset);
             if (leftBiasedWord &&
                 leftBiasedWord.startColumn <= _position.column &&
@@ -214,30 +261,25 @@ class TokenizationTextModelPart extends TextModelPart {
         };
         this._languageId = languageId;
         this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
-        this.grammarTokens.resetTokenization();
+        this._tokens.resetTokenization();
+        this.createPreferredTokenProvider();
         this._onDidChangeLanguage.fire(e);
         this._onDidChangeLanguageConfiguration.fire({});
     }
-}
-class GrammarTokens extends Disposable {
-    get backgroundTokenizationState() {
-        return this._backgroundTokenizationState;
-    }
-    constructor(_languageIdCodec, _textModel, getLanguageId, attachedViews) {
-        super();
-        this._languageIdCodec = _languageIdCodec;
-        this._textModel = _textModel;
-        this.getLanguageId = getLanguageId;
+};
+TokenizationTextModelPart = TokenizationTextModelPart_1 = ( __decorate([
+    ( __param(4, ILanguageService)),
+    ( __param(5, ILanguageConfigurationService)),
+    ( __param(6, ITreeSitterParserService))
+], TokenizationTextModelPart));
+class GrammarTokens extends AbstractTokens {
+    constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
+        super(languageIdCodec, textModel, getLanguageId);
         this._tokenizer = null;
         this._defaultBackgroundTokenizer = null;
         this._backgroundTokenizer = this._register(( new MutableDisposable()));
         this._tokens = ( new ContiguousTokensStore(this._languageIdCodec));
         this._debugBackgroundTokenizer = this._register(( new MutableDisposable()));
-        this._backgroundTokenizationState = BackgroundTokenizationState.InProgress;
-        this._onDidChangeBackgroundTokenizationState = this._register(( new Emitter()));
-        this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
-        this._onDidChangeTokens = this._register(( new Emitter()));
-        this.onDidChangeTokens = this._onDidChangeTokens.event;
         this._attachedViewStates = this._register(( new DisposableMap()));
         this._register(TokenizationRegistry.onDidChange((e) => {
             const languageId = this.getLanguageId();
@@ -434,11 +476,6 @@ class GrammarTokens extends Disposable {
         }
         return this._tokenizer.isCheapToTokenize(lineNumber);
     }
-    tokenizeIfCheap(lineNumber) {
-        if (this.isCheapToTokenize(lineNumber)) {
-            this.forceTokenization(lineNumber);
-        }
-    }
     getLineTokens(lineNumber) {
         const lineText = this._textModel.getLineContent(lineNumber);
         const result = this._tokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
@@ -472,31 +509,4 @@ class GrammarTokens extends Disposable {
         return this._tokens.hasTokens;
     }
 }
-class AttachedViewHandler extends Disposable {
-    get lineRanges() { return this._lineRanges; }
-    constructor(_refreshTokens) {
-        super();
-        this._refreshTokens = _refreshTokens;
-        this.runner = this._register(( new RunOnceScheduler(() => this.update(), 50)));
-        this._computedLineRanges = [];
-        this._lineRanges = [];
-    }
-    update() {
-        if (equals(this._computedLineRanges, this._lineRanges, (a, b) => a.equals(b))) {
-            return;
-        }
-        this._computedLineRanges = this._lineRanges;
-        this._refreshTokens();
-    }
-    handleStateChange(state) {
-        this._lineRanges = state.visibleLineRanges;
-        if (state.stabilized) {
-            this.runner.cancel();
-            this.update();
-        }
-        else {
-            this.runner.schedule();
-        }
-    }
-}
 export { TokenizationTextModelPart };

package/vscode/src/vs/editor/common/model/tokens.js

@@ -0,0 +1,82 @@
+import { equals } from '../../../base/common/arrays.js';
+import { RunOnceScheduler } from '../../../base/common/async.js';
+import { Emitter } from '../../../base/common/event.js';
+import { Disposable } from '../../../base/common/lifecycle.js';
+import { LineRange } from '../core/lineRange.js';
+import { BackgroundTokenizationState } from '../tokenizationTextModelPart.js';
+class AttachedViews {
+    constructor() {
+        this._onDidChangeVisibleRanges = ( new Emitter());
+        this.onDidChangeVisibleRanges = this._onDidChangeVisibleRanges.event;
+        this._views = ( new Set());
+    }
+    attachView() {
+        const view = ( new AttachedViewImpl((state) => {
+            this._onDidChangeVisibleRanges.fire({ view, state });
+        }));
+        this._views.add(view);
+        return view;
+    }
+    detachView(view) {
+        this._views.delete(view);
+        this._onDidChangeVisibleRanges.fire({ view, state: undefined });
+    }
+}
+class AttachedViewImpl {
+    constructor(handleStateChange) {
+        this.handleStateChange = handleStateChange;
+    }
+    setVisibleLines(visibleLines, stabilized) {
+        const visibleLineRanges = ( visibleLines.map((line) => ( new LineRange(line.startLineNumber, line.endLineNumber + 1))));
+        this.handleStateChange({ visibleLineRanges, stabilized });
+    }
+}
+class AttachedViewHandler extends Disposable {
+    get lineRanges() { return this._lineRanges; }
+    constructor(_refreshTokens) {
+        super();
+        this._refreshTokens = _refreshTokens;
+        this.runner = this._register(( new RunOnceScheduler(() => this.update(), 50)));
+        this._computedLineRanges = [];
+        this._lineRanges = [];
+    }
+    update() {
+        if (equals(this._computedLineRanges, this._lineRanges, (a, b) => a.equals(b))) {
+            return;
+        }
+        this._computedLineRanges = this._lineRanges;
+        this._refreshTokens();
+    }
+    handleStateChange(state) {
+        this._lineRanges = state.visibleLineRanges;
+        if (state.stabilized) {
+            this.runner.cancel();
+            this.update();
+        }
+        else {
+            this.runner.schedule();
+        }
+    }
+}
+class AbstractTokens extends Disposable {
+    get backgroundTokenizationState() {
+        return this._backgroundTokenizationState;
+    }
+    constructor(_languageIdCodec, _textModel, getLanguageId) {
+        super();
+        this._languageIdCodec = _languageIdCodec;
+        this._textModel = _textModel;
+        this.getLanguageId = getLanguageId;
+        this._backgroundTokenizationState = BackgroundTokenizationState.InProgress;
+        this._onDidChangeBackgroundTokenizationState = this._register(( new Emitter()));
+        this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
+        this._onDidChangeTokens = this._register(( new Emitter()));
+        this.onDidChangeTokens = this._onDidChangeTokens.event;
+    }
+    tokenizeIfCheap(lineNumber) {
+        if (this.isCheapToTokenize(lineNumber)) {
+            this.forceTokenization(lineNumber);
+        }
+    }
+}
+export { AbstractTokens, AttachedViewHandler, AttachedViews };

package/vscode/src/vs/editor/common/model/treeSitterTokens.js

@@ -0,0 +1,69 @@
+import { TreeSitterTokenizationRegistry } from '../languages.js';
+import { LineTokens } from '../tokens/lineTokens.js';
+import { StandardTokenType } from '../encodedTokenAttributes.js';
+import { AbstractTokens } from './tokens.js';
+class TreeSitterTokens extends AbstractTokens {
+    constructor(_treeSitterService, languageIdCodec, textModel, languageId) {
+        super(languageIdCodec, textModel, languageId);
+        this._treeSitterService = _treeSitterService;
+        this._tokenizationSupport = null;
+        this._initialize();
+    }
+    _initialize() {
+        const newLanguage = this.getLanguageId();
+        if (!this._tokenizationSupport || this._lastLanguageId !== newLanguage) {
+            this._lastLanguageId = newLanguage;
+            this._tokenizationSupport = TreeSitterTokenizationRegistry.get(newLanguage);
+        }
+    }
+    getLineTokens(lineNumber) {
+        const content = this._textModel.getLineContent(lineNumber);
+        if (this._tokenizationSupport) {
+            const rawTokens = this._tokenizationSupport.tokenizeEncoded(lineNumber, this._textModel);
+            if (rawTokens) {
+                return ( new LineTokens(rawTokens, content, this._languageIdCodec));
+            }
+        }
+        return LineTokens.createEmpty(content, this._languageIdCodec);
+    }
+    resetTokenization(fireTokenChangeEvent = true) {
+        if (fireTokenChangeEvent) {
+            this._onDidChangeTokens.fire({
+                semanticTokensApplied: false,
+                ranges: [
+                    {
+                        fromLineNumber: 1,
+                        toLineNumber: this._textModel.getLineCount(),
+                    },
+                ],
+            });
+        }
+        this._initialize();
+    }
+    handleDidChangeAttached() {
+    }
+    handleDidChangeContent(e) {
+        if (e.isFlush) {
+            this.resetTokenization(false);
+        }
+    }
+    forceTokenization(lineNumber) {
+    }
+    hasAccurateTokensForLine(lineNumber) {
+        return true;
+    }
+    isCheapToTokenize(lineNumber) {
+        return true;
+    }
+    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
+        return StandardTokenType.Other;
+    }
+    tokenizeLineWithEdit(position, length, newText) {
+        return null;
+    }
+    get hasTokens() {
+        const hasTree = this._treeSitterService.getParseResult(this._textModel) !== undefined;
+        return hasTree;
+    }
+}
+export { TreeSitterTokens };

package/vscode/src/vs/editor/common/model.d.ts

@@ -507,6 +507,8 @@ interface IReadonlyTextBuffer {
     getLineCharCode(lineNumber: number, index: number): number;
     getCharCode(offset: number): number;
     getLineLength(lineNumber: number): number;
+    getLineMinColumn(lineNumber: number): number;
+    getLineMaxColumn(lineNumber: number): number;
     getLineFirstNonWhitespaceColumn(lineNumber: number): number;
     getLineLastNonWhitespaceColumn(lineNumber: number): number;
     findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[];