@codingame/monaco-vscode-treesitter-service-override 15.0.2 → 16.0.0

package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.js

@@ -16,13 +16,18 @@ import { StopWatch } from '@codingame/monaco-vscode-api/vscode/vs/base/common/st
 import { ITreeSitterTokenizationStoreService } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/treeSitterTokenStoreService.service';
 import { TokenQuality } from '../../../../editor/common/model/tokenStore.js';
 import { Range } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range';
-import { ICodeEditorService } from '@codingame/monaco-vscode-api/vscode/vs/editor/browser/services/codeEditorService.service';
 import { setTimeout0 } from '@codingame/monaco-vscode-api/vscode/vs/base/common/platform';
 import { findLikelyRelevantLines } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/textModelTokens';
 import { TreeSitterCodeEditors } from './treeSitterCodeEditors.js';
-import { Position } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/position';
 import { IWorkbenchThemeService } from '@codingame/monaco-vscode-7ba0af96-90c2-5e11-ad7f-befdbbf246c8-common/vscode/vs/workbench/services/themes/common/workbenchThemeService.service';
+import { Position } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/position';
 
+const TREESITTER_BASE_SCOPES = {
+    'css': 'source.css',
+    'typescript': 'source.ts',
+    'ini': 'source.ini',
+    'regex': 'source.regex',
+};
 const BRACKETS = /[\{\}\[\]\<\>\(\)]/g;
 let TreeSitterTokenizationFeature = class TreeSitterTokenizationFeature extends Disposable {
     constructor(_treeSitterImporter, _languageService, _configurationService, _instantiationService, _fileService) {
@@ -79,7 +84,7 @@ TreeSitterTokenizationFeature = ( __decorate([
     ( __param(4, IFileService))
 ], TreeSitterTokenizationFeature));
 let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends Disposable {
-    constructor(_queries, Query, _languageId, _languageIdCodec, _treeSitterService, _themeService, _tokenizationStoreService, _codeEditorService, _instantiationService) {
+    constructor(_queries, Query, _languageId, _languageIdCodec, _treeSitterService, _themeService, _tokenizationStoreService, _instantiationService) {
         super();
         this._queries = _queries;
         this.Query = Query;
@@ -88,12 +93,11 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
         this._treeSitterService = _treeSitterService;
         this._themeService = _themeService;
         this._tokenizationStoreService = _tokenizationStoreService;
-        this._codeEditorService = _codeEditorService;
         this._instantiationService = _instantiationService;
         this._onDidChangeTokens = this._register(( new Emitter()));
         this.onDidChangeTokens = this._onDidChangeTokens.event;
-        this._onDidCompleteFirstTokenization = this._register(( new Emitter()));
-        this.onDidCompleteFirstTokenization = this._onDidCompleteFirstTokenization.event;
+        this._onDidCompleteBackgroundTokenization = this._register(( new Emitter()));
+        this.onDidChangeBackgroundTokenization = this._onDidCompleteBackgroundTokenization.event;
         this._codeEditors = this._instantiationService.createInstance(TreeSitterCodeEditors, this._languageId);
         this._register(this._codeEditors.onDidChangeViewport(e => {
             this._parseAndTokenizeViewPort(e.model, e.ranges);
@@ -104,9 +108,8 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             }
         });
         this._register(Event.runAndSubscribe(this._themeService.onDidColorThemeChange, (e) => this._updateTheme(e)));
-        let hasDoneFullTokenization = false;
         this._register(this._treeSitterService.onDidUpdateTree((e) => {
-            if (e.textModel.getLanguageId() !== this._languageId) {
+            if (e.languageId !== this._languageId) {
                 return;
             }
             if (this._tokenizationStoreService.hasTokens(e.textModel)) {
@@ -117,27 +120,25 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             if (e.versionId !== e.textModel.getVersionId()) {
                 return;
             }
-            let updatePromise;
             if (!this._tokenizationStoreService.hasTokens(e.textModel)) {
-                updatePromise = this._firstTreeUpdate(e.textModel, e.versionId);
+                this._firstTreeUpdate(e.textModel, e.versionId, e.tree);
             }
             else {
-                updatePromise = this._handleTreeUpdate(e);
-            }
-            if (!hasDoneFullTokenization) {
-                hasDoneFullTokenization = true;
-                updatePromise.then(() => {
-                    this._onDidCompleteFirstTokenization.fire({ textModel: e.textModel });
-                });
+                this._handleTreeUpdate(e.ranges, e.textModel, e.versionId, e.tree);
             }
         }));
     }
+    get _encodedLanguageId() {
+        if (!this._encodedLanguage) {
+            this._encodedLanguage = this._languageIdCodec.encodeLanguageId(this._languageId);
+        }
+        return this._encodedLanguage;
+    }
     _setInitialTokens(textModel) {
         const tokens = this._createEmptyTokens(textModel);
         this._tokenizationStoreService.setTokens(textModel, tokens, TokenQuality.None);
     }
-    _parseAndTokenizeViewPortRange(model, range, languageId, startOffsetOfRangeInDocument, endOffsetOfRangeInDocument) {
-        const content = model.getValueInRange(range);
+    _forceParseAndTokenizeContent(model, range, startOffsetOfRangeInDocument, endOffsetOfRangeInDocument, content, asUpdate) {
         const likelyRelevantLines = findLikelyRelevantLines(model, range.startLineNumber).likelyRelevantLines;
         const likelyRelevantPrefix = likelyRelevantLines.join(model.getEOL());
         const tree = this._treeSitterService.getTreeSync(`${likelyRelevantPrefix}${content}`, this._languageId);
@@ -151,17 +152,21 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             range.endColumn
         ));
         const captures = this._captureAtRange(treeRange, tree);
-        const tokens = this._tokenizeCapturesWithMetadata(tree, captures, languageId, likelyRelevantPrefix.length, endOffsetOfRangeInDocument - startOffsetOfRangeInDocument);
+        const tokens = this._tokenizeCapturesWithMetadata(tree, captures, likelyRelevantPrefix.length, endOffsetOfRangeInDocument - startOffsetOfRangeInDocument + likelyRelevantPrefix.length);
         if (!tokens) {
             return;
         }
-        return this._rangeTokensAsUpdates(startOffsetOfRangeInDocument, tokens.endOffsetsAndMetadata, likelyRelevantPrefix.length);
+        if (asUpdate) {
+            return this._rangeTokensAsUpdates(startOffsetOfRangeInDocument, tokens.endOffsetsAndMetadata, likelyRelevantPrefix.length);
+        }
+        else {
+            return tokens.endOffsetsAndMetadata;
+        }
     }
     async _parseAndTokenizeViewPort(model, viewportRanges) {
         if (!this._tokenizationStoreService.hasTokens(model)) {
             this._setInitialTokens(model);
         }
-        const languageId = this._languageIdCodec.encodeLanguageId(this._languageId);
         for (const range of viewportRanges) {
             const startOffsetOfRangeInDocument = model.getOffsetAt(range.getStartPosition());
             const endOffsetOfRangeInDocument = model.getOffsetAt(range.getEndPosition());
@@ -169,7 +174,8 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             if (this._tokenizationStoreService.rangeHasTokens(model, range, TokenQuality.ViewportGuess)) {
                 continue;
             }
-            const tokenUpdates = await this._parseAndTokenizeViewPortRange(model, range, languageId, startOffsetOfRangeInDocument, endOffsetOfRangeInDocument);
+            const content = model.getValueInRange(range);
+            const tokenUpdates = await this._forceParseAndTokenizeContent(model, range, startOffsetOfRangeInDocument, endOffsetOfRangeInDocument, content, true);
             if (!tokenUpdates || this._tokenizationStoreService.rangeHasTokens(model, range, TokenQuality.ViewportGuess)) {
                 continue;
             }
@@ -182,27 +188,64 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             this._onDidChangeTokens.fire({ textModel: model, changes: { semanticTokensApplied: false, ranges: [{ fromLineNumber: range.startLineNumber, toLineNumber: range.endLineNumber }] } });
         }
     }
+    guessTokensForLinesContent(lineNumber, textModel, lines) {
+        if (lines.length === 0) {
+            return undefined;
+        }
+        const lineContent = lines.join(textModel.getEOL());
+        const range = ( new Range(1, 1, lineNumber + lines.length, lines[lines.length - 1].length + 1));
+        const startOffset = textModel.getOffsetAt({ lineNumber, column: 1 });
+        const tokens = this._forceParseAndTokenizeContent(textModel, range, startOffset, startOffset + lineContent.length, lineContent, false);
+        if (!tokens) {
+            return undefined;
+        }
+        const tokensByLine = ( new Array(lines.length));
+        let tokensIndex = 0;
+        let tokenStartOffset = 0;
+        let lineStartOffset = 0;
+        for (let i = 0; i < lines.length; i++) {
+            const tokensForLine = [];
+            let moveToNextLine = false;
+            for (let j = tokensIndex; (!moveToNextLine && (j < tokens.length)); j++) {
+                const token = tokens[j];
+                const lineAdjustedEndOffset = token.endOffset - lineStartOffset;
+                const lineAdjustedStartOffset = tokenStartOffset - lineStartOffset;
+                if (lineAdjustedEndOffset <= lines[i].length) {
+                    tokensForLine.push({ endOffset: lineAdjustedEndOffset, metadata: token.metadata });
+                    tokensIndex++;
+                }
+                else if (lineAdjustedStartOffset < lines[i].length) {
+                    const partialToken = { endOffset: lines[i].length, metadata: token.metadata };
+                    tokensForLine.push(partialToken);
+                    moveToNextLine = true;
+                }
+                else {
+                    moveToNextLine = true;
+                }
+                tokenStartOffset = token.endOffset;
+            }
+            tokensByLine[i] = this._endOffsetTokensToUint32Array(tokensForLine);
+            lineStartOffset += lines[i].length + textModel.getEOL().length;
+        }
+        return tokensByLine;
+    }
     _emptyTokensForOffsetAndLength(offset, length, emptyToken) {
         return { token: emptyToken, length: offset + length, startOffsetInclusive: 0 };
     }
     _createEmptyTokens(textModel) {
-        const languageId = this._languageIdCodec.encodeLanguageId(this._languageId);
-        const emptyToken = this._emptyToken(languageId);
+        const emptyToken = this._emptyToken();
         const modelEndOffset = textModel.getValueLength();
         const emptyTokens = [this._emptyTokensForOffsetAndLength(0, modelEndOffset, emptyToken)];
         return emptyTokens;
     }
-    _firstTreeUpdate(textModel, versionId) {
+    _firstTreeUpdate(textModel, versionId, tree) {
         this._setInitialTokens(textModel);
-        return this._setViewPortTokens(textModel, versionId);
+        return this._setViewPortTokens(textModel, versionId, tree);
     }
-    _codeEditorForModel(textModel) {
-        return this._codeEditorService.listCodeEditors().find(editor => editor.getModel() === textModel);
-    }
-    _setViewPortTokens(textModel, versionId) {
+    _setViewPortTokens(textModel, versionId, tree) {
         const maxLine = textModel.getLineCount();
         let rangeChanges;
-        const editor = this._codeEditorForModel(textModel);
+        const editor = this._codeEditors.getEditorForModel(textModel);
         if (editor) {
             const viewPort = editor.getVisibleRangesPlusViewportAboveBelow();
             const ranges = ( new Array(viewPort.length));
@@ -216,35 +259,41 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
                     newRange: range,
                     newRangeStartOffset,
                     newRangeEndOffset,
-                    oldRangeLength: newRangeEndOffset - newRangeStartOffset
                 };
             }
         }
         else {
             const valueLength = textModel.getValueLength();
-            rangeChanges = [{ newRange: ( new Range(1, 1, maxLine, textModel.getLineMaxColumn(maxLine))), newRangeStartOffset: 0, newRangeEndOffset: valueLength, oldRangeLength: valueLength }];
+            rangeChanges = [{ newRange: ( new Range(1, 1, maxLine, textModel.getLineMaxColumn(maxLine))), newRangeStartOffset: 0, newRangeEndOffset: valueLength }];
         }
-        return this._handleTreeUpdate({ ranges: rangeChanges, textModel, versionId });
+        return this._handleTreeUpdate(rangeChanges, textModel, versionId, tree);
     }
-    _handleTreeUpdate(e) {
+    _handleTreeUpdate(ranges, textModel, versionId, textModelTreeSitter) {
+        const tree = textModelTreeSitter.parseResult?.tree;
+        if (!tree) {
+            return;
+        }
         const rangeChanges = [];
         const chunkSize = 1000;
-        for (let i = 0; i < e.ranges.length; i++) {
-            const rangeLinesLength = e.ranges[i].newRange.endLineNumber - e.ranges[i].newRange.startLineNumber;
+        for (let i = 0; i < ranges.length; i++) {
+            const rangeLinesLength = ranges[i].newRange.endLineNumber - ranges[i].newRange.startLineNumber;
             if (rangeLinesLength > chunkSize) {
-                const fullRangeEndLineNumber = e.ranges[i].newRange.endLineNumber;
-                let chunkLineStart = e.ranges[i].newRange.startLineNumber;
+                const fullRangeEndLineNumber = ranges[i].newRange.endLineNumber;
+                let chunkLineStart = ranges[i].newRange.startLineNumber;
+                let chunkColumnStart = ranges[i].newRange.startColumn;
                 let chunkLineEnd = chunkLineStart + chunkSize;
                 do {
-                    const chunkStartingPosition = ( new Position(chunkLineStart, 1));
-                    const chunkEndPosition = ( new Position(chunkLineEnd, e.textModel.getLineMaxColumn(chunkLineEnd)));
+                    const chunkStartingPosition = ( new Position(chunkLineStart, chunkColumnStart));
+                    const chunkEndColumn = ((chunkLineEnd === ranges[i].newRange.endLineNumber) ? ranges[i].newRange.endColumn : textModel.getLineMaxColumn(chunkLineEnd));
+                    const chunkEndPosition = ( new Position(chunkLineEnd, chunkEndColumn));
                     const chunkRange = Range.fromPositions(chunkStartingPosition, chunkEndPosition);
                     rangeChanges.push({
                         range: chunkRange,
-                        startOffset: e.textModel.getOffsetAt(chunkRange.getStartPosition()),
-                        endOffset: e.textModel.getOffsetAt(chunkRange.getEndPosition())
+                        startOffset: textModel.getOffsetAt(chunkRange.getStartPosition()),
+                        endOffset: textModel.getOffsetAt(chunkRange.getEndPosition())
                     });
                     chunkLineStart = chunkLineEnd + 1;
+                    chunkColumnStart = 1;
                     if (chunkLineEnd < fullRangeEndLineNumber && chunkLineEnd + chunkSize > fullRangeEndLineNumber) {
                         chunkLineEnd = fullRangeEndLineNumber;
                     }
@@ -254,51 +303,46 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
                 } while (chunkLineEnd <= fullRangeEndLineNumber);
             }
             else {
-                if ((i === 0) || (rangeChanges[i - 1].range.endLineNumber < e.ranges[i].newRange.startLineNumber)) {
-                    const range = ( new Range(
-                        e.ranges[i].newRange.startLineNumber,
-                        1,
-                        e.ranges[i].newRange.endLineNumber,
-                        e.textModel.getLineMaxColumn(e.ranges[i].newRange.endLineNumber)
-                    ));
+                if ((i === 0) || (rangeChanges[i - 1].endOffset < ranges[i].newRangeStartOffset)) {
                     rangeChanges.push({
-                        range,
-                        startOffset: e.textModel.getOffsetAt(range.getStartPosition()),
-                        endOffset: e.textModel.getOffsetAt(range.getEndPosition())
+                        range: ranges[i].newRange,
+                        startOffset: ranges[i].newRangeStartOffset,
+                        endOffset: ranges[i].newRangeEndOffset
                     });
                 }
-                else if (rangeChanges[i - 1].range.endLineNumber < e.ranges[i].newRange.endLineNumber) {
+                else if (rangeChanges[i - 1].endOffset < ranges[i].newRangeEndOffset) {
+                    const startPosition = textModel.getPositionAt(rangeChanges[i - 1].endOffset + 1);
                     const range = ( new Range(
-                        rangeChanges[i - 1].range.endLineNumber + 1,
-                        1,
-                        e.ranges[i].newRange.endLineNumber,
-                        e.textModel.getLineMaxColumn(e.ranges[i].newRange.endLineNumber)
+                        startPosition.lineNumber,
+                        startPosition.column,
+                        ranges[i].newRange.endLineNumber,
+                        ranges[i].newRange.endColumn
                     ));
                     rangeChanges.push({
                         range,
-                        startOffset: e.textModel.getOffsetAt(range.getStartPosition()),
-                        endOffset: e.textModel.getOffsetAt(range.getEndPosition())
+                        startOffset: rangeChanges[i - 1].endOffset + 1,
+                        endOffset: ranges[i].newRangeEndOffset
                     });
                 }
             }
         }
-        const captures = ( rangeChanges.map(range => this._getTreeAndCaptures(range.range, e.textModel)));
-        return this._updateTreeForRanges(e.textModel, rangeChanges, e.versionId, captures).then(() => {
-            const tree = this._getTree(e.textModel);
-            if (!e.textModel.isDisposed() && (tree?.versionId === e.textModel.getVersionId())) {
-                this._refreshNeedsRefresh(e.textModel, e.versionId);
+        const captures = ( rangeChanges.map(range => this._getCaptures(range.range, textModelTreeSitter, tree)));
+        return this._updateTreeForRanges(textModel, rangeChanges, versionId, tree, captures).then(() => {
+            const tree = this._getTree(textModel);
+            if (!textModel.isDisposed() && (tree?.parseResult?.versionId === textModel.getVersionId())) {
+                this._refreshNeedsRefresh(textModel, versionId);
             }
         });
     }
-    async _updateTreeForRanges(textModel, rangeChanges, versionId, captures) {
+    async _updateTreeForRanges(textModel, rangeChanges, versionId, tree, captures) {
         let tokenUpdate;
         for (let i = 0; i < rangeChanges.length; i++) {
-            if (versionId !== textModel.getVersionId()) {
+            if (!textModel.isDisposed() && versionId !== textModel.getVersionId()) {
                 break;
             }
             const capture = captures[i];
             const range = rangeChanges[i];
-            const updates = this.getTokensInRange(textModel, range.range, range.startOffset, range.endOffset, capture);
+            const updates = this.getTokensInRange(textModel, range.range, range.startOffset, range.endOffset, tree, capture);
             if (updates) {
                 tokenUpdate = { newTokens: updates };
             }
@@ -315,6 +359,7 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             });
             await ( new Promise(resolve => setTimeout0(resolve)));
         }
+        this._onDidCompleteBackgroundTokenization.fire({ textModel });
     }
     _refreshNeedsRefresh(textModel, versionId) {
         const rangesToRefresh = this._tokenizationStoreService.getNeedsRefresh(textModel);
@@ -327,11 +372,13 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             rangeChanges[i] = {
                 newRange: range.range,
                 newRangeStartOffset: range.startOffset,
-                newRangeEndOffset: range.endOffset,
-                oldRangeLength: range.endOffset - range.startOffset
+                newRangeEndOffset: range.endOffset
             };
         }
-        this._handleTreeUpdate({ ranges: rangeChanges, textModel, versionId });
+        const tree = this._getTree(textModel);
+        if (tree?.parseResult?.tree && tree.parseResult.versionId === versionId) {
+            this._handleTreeUpdate(rangeChanges, textModel, versionId, tree);
+        }
     }
     _rangeTokensAsUpdates(rangeOffset, endOffsetToken, startingOffsetInArray) {
         const updates = [];
@@ -352,9 +399,8 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
         }
         return updates;
     }
-    getTokensInRange(textModel, range, rangeStartOffset, rangeEndOffset, captures) {
-        const languageId = this._languageIdCodec.encodeLanguageId(this._languageId);
-        const tokens = captures ? this._tokenizeCapturesWithMetadata(captures.tree?.tree, captures.captures, languageId, rangeStartOffset, rangeEndOffset) : this._tokenize(languageId, range, rangeStartOffset, rangeEndOffset, textModel);
+    getTokensInRange(textModel, range, rangeStartOffset, rangeEndOffset, tree, captures) {
+        const tokens = captures ? this._tokenizeCapturesWithMetadata(tree, captures, rangeStartOffset, rangeEndOffset) : this._tokenize(range, rangeStartOffset, rangeEndOffset, textModel);
         if (tokens?.endOffsetsAndMetadata) {
             return this._rangeTokensAsUpdates(rangeStartOffset, tokens.endOffsetsAndMetadata);
         }
@@ -380,34 +426,25 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
     }
     _updateTheme(e) {
         this._colorThemeData = this._themeService.getColorTheme();
-        for (const editor of this._codeEditors.editors) {
-            const model = editor.getModel();
-            if (model) {
-                const modelRange = model.getFullModelRange();
-                this._tokenizationStoreService.markForRefresh(model, modelRange);
+        for (const model of this._codeEditors.textModels) {
+            const modelRange = model.getFullModelRange();
+            this._tokenizationStoreService.markForRefresh(model, modelRange);
+            const editor = this._codeEditors.getEditorForModel(model);
+            if (editor) {
                 this._parseAndTokenizeViewPort(model, editor.getVisibleRangesPlusViewportAboveBelow());
-                if (e?.target !== 'preview') {
-                    this._handleTreeUpdate({
-                        ranges: [{
-                                newRange: modelRange,
-                                newRangeStartOffset: 0,
-                                newRangeEndOffset: model.getValueLength(),
-                                oldRangeLength: model.getValueLength()
-                            }],
-                        textModel: model,
-                        versionId: model.getVersionId()
-                    });
-                }
             }
         }
     }
     captureAtPosition(lineNumber, column, textModel) {
-        const tree = this._getTree(textModel);
-        const captures = this._captureAtRange(( new Range(lineNumber, column, lineNumber, column + 1)), tree?.tree);
+        const textModelTreeSitter = this._getTree(textModel);
+        if (!textModelTreeSitter?.parseResult?.tree) {
+            return [];
+        }
+        const captures = this._captureAtRangeWithInjections(( new Range(lineNumber, column, lineNumber, column + 1)), textModelTreeSitter, textModelTreeSitter.parseResult.tree);
         return captures;
     }
-    captureAtPositionTree(lineNumber, column, tree) {
-        const captures = this._captureAtRange(( new Range(lineNumber, column, lineNumber, column + 1)), tree);
+    captureAtRangeTree(range, tree, textModelTreeSitter) {
+        const captures = textModelTreeSitter ? this._captureAtRangeWithInjections(range, textModelTreeSitter, tree) : this._captureAtRange(range, tree);
         return captures;
     }
     _captureAtRange(range, tree) {
@@ -420,10 +457,44 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             text: capture.node.text,
             node: {
                 startIndex: capture.node.startIndex,
-                endIndex: capture.node.endIndex
-            }
+                endIndex: capture.node.endIndex,
+                startPosition: {
+                    lineNumber: capture.node.startPosition.row + 1,
+                    column: capture.node.startPosition.column + 1
+                },
+                endPosition: {
+                    lineNumber: capture.node.endPosition.row + 1,
+                    column: capture.node.endPosition.column + 1
+                }
+            },
+            encodedLanguageId: this._encodedLanguageId
         })));
     }
+    _captureAtRangeWithInjections(range, textModelTreeSitter, tree) {
+        const query = this._ensureQuery();
+        if (!textModelTreeSitter?.parseResult || !query) {
+            return [];
+        }
+        const captures = this._captureAtRange(range, tree);
+        for (let i = 0; i < captures.length; i++) {
+            const capture = captures[i];
+            const capStartLine = capture.node.startPosition.lineNumber;
+            const capEndLine = capture.node.endPosition.lineNumber;
+            const capStartColumn = capture.node.startPosition.column;
+            const capEndColumn = capture.node.endPosition.column;
+            const startLine = ((capStartLine > range.startLineNumber) && (capStartLine < range.endLineNumber)) ? capStartLine : range.startLineNumber;
+            const endLine = ((capEndLine > range.startLineNumber) && (capEndLine < range.endLineNumber)) ? capEndLine : range.endLineNumber;
+            const startColumn = (capStartLine === range.startLineNumber) ? (capStartColumn < range.startColumn ? range.startColumn : capStartColumn) : (capStartLine < range.startLineNumber ? range.startColumn : capStartColumn);
+            const endColumn = (capEndLine === range.endLineNumber) ? (capEndColumn > range.endColumn ? range.endColumn : capEndColumn) : (capEndLine > range.endLineNumber ? range.endColumn : capEndColumn);
+            const injectionRange = ( new Range(startLine, startColumn, endLine, endColumn));
+            const injection = this._getInjectionCaptures(textModelTreeSitter, capture, injectionRange);
+            if (injection && injection.length > 0) {
+                captures.splice(i + 1, 0, ...injection);
+                i += injection.length;
+            }
+        }
+        return captures;
+    }
     tokenizeEncoded(lineNumber, textModel) {
         const tokens = this._tokenizeEncoded(lineNumber, textModel);
         if (!tokens) {
@@ -441,35 +512,85 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
         }
         return { result: this._endOffsetTokensToUint32Array(tokens.result), captureTime: tokens.captureTime, metadataTime: tokens.metadataTime };
     }
-    _getTreeAndCaptures(range, textModel) {
-        const tree = this._getTree(textModel);
-        const captures = this._captureAtRange(range, tree?.tree);
-        return { tree, captures };
+    _getCaptures(range, textModelTreeSitter, tree) {
+        const captures = this._captureAtRangeWithInjections(range, textModelTreeSitter, tree);
+        return captures;
     }
-    _tokenize(encodedLanguageId, range, rangeStartOffset, rangeEndOffset, textModel) {
-        const { tree, captures } = this._getTreeAndCaptures(range, textModel);
-        const result = this._tokenizeCapturesWithMetadata(tree?.tree, captures, encodedLanguageId, rangeStartOffset, rangeEndOffset);
-        if (!tree || !result) {
+    _tokenize(range, rangeStartOffset, rangeEndOffset, textModel) {
+        const tree = this._getTree(textModel);
+        if (!tree?.parseResult?.tree) {
             return undefined;
         }
-        return { ...result, versionId: tree.versionId };
+        const captures = this._getCaptures(range, tree, tree.parseResult.tree);
+        const result = this._tokenizeCapturesWithMetadata(tree.parseResult.tree, captures, rangeStartOffset, rangeEndOffset);
+        if (!result) {
+            return undefined;
+        }
+        return { ...result, versionId: tree.parseResult.versionId };
     }
     _createTokensFromCaptures(tree, captures, rangeStartOffset, rangeEndOffset) {
         const stopwatch = StopWatch.create();
         const rangeLength = rangeEndOffset - rangeStartOffset;
+        const encodedLanguageId = this._languageIdCodec.encodeLanguageId(this._languageId);
+        const baseScope = TREESITTER_BASE_SCOPES[this._languageId] || 'source';
         if (captures.length === 0) {
             if (tree) {
                 stopwatch.stop();
-                const endOffsetsAndMetadata = [{ endOffset: rangeLength, scopes: [] }];
+                const endOffsetsAndMetadata = [{ endOffset: rangeLength, scopes: [], encodedLanguageId }];
                 return { endOffsets: endOffsetsAndMetadata, captureTime: stopwatch.elapsed() };
             }
             return undefined;
         }
         const endOffsetsAndScopes = Array(captures.length);
-        endOffsetsAndScopes.fill({ endOffset: 0, scopes: [] });
+        endOffsetsAndScopes.fill({ endOffset: 0, scopes: [baseScope], encodedLanguageId });
         let tokenIndex = 0;
         const increaseSizeOfTokensByOneToken = () => {
-            endOffsetsAndScopes.push({ endOffset: 0, scopes: [] });
+            endOffsetsAndScopes.push({ endOffset: 0, scopes: [baseScope], encodedLanguageId });
+        };
+        const brackets = (capture, startOffset) => {
+            return (capture.name.includes('punctuation') && capture.text) ? ( Array.from(capture.text.matchAll(BRACKETS)).map(match => startOffset + match.index)) : undefined;
+        };
+        const addCurrentTokenToArray = (capture, startOffset, endOffset, position) => {
+            if (position !== undefined) {
+                const oldScopes = endOffsetsAndScopes[position].scopes;
+                let oldBracket = endOffsetsAndScopes[position].bracket;
+                const prevEndOffset = position > 0 ? endOffsetsAndScopes[position - 1].endOffset : 0;
+                if (prevEndOffset !== startOffset) {
+                    let preInsertBracket = undefined;
+                    if (oldBracket && oldBracket.length > 0) {
+                        preInsertBracket = [];
+                        const postInsertBracket = [];
+                        for (let i = 0; i < oldBracket.length; i++) {
+                            const bracket = oldBracket[i];
+                            if (bracket < startOffset) {
+                                preInsertBracket.push(bracket);
+                            }
+                            else if (bracket > endOffset) {
+                                postInsertBracket.push(bracket);
+                            }
+                        }
+                        if (preInsertBracket.length === 0) {
+                            preInsertBracket = undefined;
+                        }
+                        if (postInsertBracket.length === 0) {
+                            oldBracket = undefined;
+                        }
+                        else {
+                            oldBracket = postInsertBracket;
+                        }
+                    }
+                    endOffsetsAndScopes.splice(position, 0, { endOffset: startOffset, scopes: [...oldScopes], bracket: preInsertBracket, encodedLanguageId: capture.encodedLanguageId });
+                    position++;
+                    increaseSizeOfTokensByOneToken();
+                    tokenIndex++;
+                }
+                endOffsetsAndScopes.splice(position, 0, { endOffset: endOffset, scopes: [...oldScopes, capture.name], bracket: brackets(capture, startOffset), encodedLanguageId: capture.encodedLanguageId });
+                endOffsetsAndScopes[tokenIndex].bracket = oldBracket;
+            }
+            else {
+                endOffsetsAndScopes[tokenIndex] = { endOffset: endOffset, scopes: [baseScope, capture.name], bracket: brackets(capture, startOffset), encodedLanguageId: capture.encodedLanguageId };
+            }
+            tokenIndex++;
         };
         for (let captureIndex = 0; captureIndex < captures.length; captureIndex++) {
             const capture = captures[captureIndex];
@@ -486,57 +607,13 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
             }
             const startOffset = endOffset - currentTokenLength;
             if ((previousEndOffset >= 0) && (previousEndOffset < startOffset)) {
-                endOffsetsAndScopes[tokenIndex] = { endOffset: startOffset, scopes: [] };
+                endOffsetsAndScopes[tokenIndex] = { endOffset: startOffset, scopes: [baseScope], encodedLanguageId: this._encodedLanguageId };
                 tokenIndex++;
                 increaseSizeOfTokensByOneToken();
             }
             if (currentTokenLength < 0) {
                 continue;
             }
-            const brackets = () => {
-                return (capture.name.includes('punctuation') && capture.text) ? ( Array.from(capture.text.matchAll(BRACKETS)).map(match => startOffset + match.index)) : undefined;
-            };
-            const addCurrentTokenToArray = (position) => {
-                if (position !== undefined) {
-                    let oldBracket = endOffsetsAndScopes[position].bracket;
-                    const prevEndOffset = position > 0 ? endOffsetsAndScopes[position - 1].endOffset : 0;
-                    if (prevEndOffset !== startOffset) {
-                        let preInsertBracket = undefined;
-                        if (oldBracket && oldBracket.length > 0) {
-                            preInsertBracket = [];
-                            const postInsertBracket = [];
-                            for (let i = 0; i < oldBracket.length; i++) {
-                                const bracket = oldBracket[i];
-                                if (bracket < startOffset) {
-                                    preInsertBracket.push(bracket);
-                                }
-                                else if (bracket > endOffset) {
-                                    postInsertBracket.push(bracket);
-                                }
-                            }
-                            if (preInsertBracket.length === 0) {
-                                preInsertBracket = undefined;
-                            }
-                            if (postInsertBracket.length === 0) {
-                                oldBracket = undefined;
-                            }
-                            else {
-                                oldBracket = postInsertBracket;
-                            }
-                        }
-                        endOffsetsAndScopes.splice(position, 0, { endOffset: startOffset, scopes: [...endOffsetsAndScopes[position].scopes], bracket: preInsertBracket });
-                        position++;
-                        increaseSizeOfTokensByOneToken();
-                        tokenIndex++;
-                    }
-                    endOffsetsAndScopes.splice(position, 0, { endOffset: endOffset, scopes: [capture.name], bracket: brackets() });
-                    endOffsetsAndScopes[tokenIndex].bracket = oldBracket;
-                }
-                else {
-                    endOffsetsAndScopes[tokenIndex] = { endOffset: endOffset, scopes: [capture.name], bracket: brackets() };
-                }
-                tokenIndex++;
-            };
             if (previousEndOffset >= endOffset) {
                 let withinTokenIndex = tokenIndex - 1;
                 let previousTokenEndOffset = endOffsetsAndScopes[withinTokenIndex].endOffset;
@@ -544,12 +621,13 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
                 do {
                     if ((previousTokenStartOffset + currentTokenLength) === previousTokenEndOffset) {
                         if (previousTokenStartOffset === startOffset) {
-                            endOffsetsAndScopes[withinTokenIndex].scopes[endOffsetsAndScopes[withinTokenIndex].scopes.length - 1] = capture.name;
-                            endOffsetsAndScopes[withinTokenIndex].bracket = brackets();
+                            endOffsetsAndScopes[withinTokenIndex].scopes.push(capture.name);
+                            const oldBracket = endOffsetsAndScopes[withinTokenIndex].bracket;
+                            endOffsetsAndScopes[withinTokenIndex].bracket = ((oldBracket && (oldBracket.length > 0)) ? oldBracket : brackets(capture, startOffset));
                         }
                     }
                     else if (previousTokenStartOffset <= startOffset) {
-                        addCurrentTokenToArray(withinTokenIndex);
+                        addCurrentTokenToArray(capture, startOffset, endOffset, withinTokenIndex);
                         break;
                     }
                     withinTokenIndex--;
@@ -558,13 +636,13 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
                 } while (previousTokenEndOffset > startOffset);
             }
             else {
-                addCurrentTokenToArray();
+                addCurrentTokenToArray(capture, startOffset, endOffset);
             }
         }
         if ((endOffsetsAndScopes[tokenIndex - 1].endOffset < rangeLength)) {
             if (rangeLength - endOffsetsAndScopes[tokenIndex - 1].endOffset > 0) {
                 increaseSizeOfTokensByOneToken();
-                endOffsetsAndScopes[tokenIndex] = { endOffset: rangeLength, scopes: endOffsetsAndScopes[tokenIndex].scopes };
+                endOffsetsAndScopes[tokenIndex] = { endOffset: rangeLength, scopes: endOffsetsAndScopes[tokenIndex].scopes, encodedLanguageId: this._encodedLanguageId };
                 tokenIndex++;
             }
         }
@@ -578,7 +656,18 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
         const captureTime = stopwatch.elapsed();
         return { endOffsets: endOffsetsAndScopes, captureTime };
     }
-    _tokenizeCapturesWithMetadata(tree, captures, encodedLanguageId, rangeStartOffset, rangeEndOffset) {
+    _getInjectionCaptures(textModelTreeSitter, parentCapture, range) {
+        const injection = textModelTreeSitter.getInjection(parentCapture.node.startIndex, this._languageId);
+        if (!injection?.tree || injection.versionId !== textModelTreeSitter.parseResult?.versionId) {
+            return undefined;
+        }
+        const feature = TreeSitterTokenizationRegistry.get(injection.languageId);
+        if (!feature) {
+            return undefined;
+        }
+        return feature.captureAtRangeTree(range, injection.tree, textModelTreeSitter);
+    }
+    _tokenizeCapturesWithMetadata(tree, captures, rangeStartOffset, rangeEndOffset) {
         const stopwatch = StopWatch.create();
         const emptyTokens = this._createTokensFromCaptures(tree, captures, rangeStartOffset, rangeEndOffset);
         if (!emptyTokens) {
@@ -587,21 +676,20 @@ let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends
         const endOffsetsAndScopes = emptyTokens.endOffsets;
         for (let i = 0; i < endOffsetsAndScopes.length; i++) {
             const token = endOffsetsAndScopes[i];
-            token.metadata = findMetadata(this._colorThemeData, token.scopes, encodedLanguageId, !!token.bracket && (token.bracket.length > 0));
+            token.metadata = findMetadata(this._colorThemeData, token.scopes, token.encodedLanguageId, !!token.bracket && (token.bracket.length > 0));
         }
         const metadataTime = stopwatch.elapsed();
         return { endOffsetsAndMetadata: endOffsetsAndScopes, captureTime: emptyTokens.captureTime, metadataTime };
     }
-    _emptyToken(encodedLanguageId) {
-        return findMetadata(this._colorThemeData, [], encodedLanguageId, false);
+    _emptyToken() {
+        return findMetadata(this._colorThemeData, [], this._encodedLanguageId, false);
     }
     _tokenizeEncoded(lineNumber, textModel) {
-        const encodedLanguageId = this._languageIdCodec.encodeLanguageId(this._languageId);
         const lineOffset = textModel.getOffsetAt({ lineNumber: lineNumber, column: 1 });
         const maxLine = textModel.getLineCount();
         const lineEndOffset = (lineNumber + 1 <= maxLine) ? textModel.getOffsetAt({ lineNumber: lineNumber + 1, column: 1 }) : textModel.getValueLength();
         const lineLength = lineEndOffset - lineOffset;
-        const result = this._tokenize(encodedLanguageId, ( new Range(lineNumber, 1, lineNumber, lineLength + 1)), lineOffset, lineEndOffset, textModel);
+        const result = this._tokenize(( new Range(lineNumber, 1, lineNumber, lineLength + 1)), lineOffset, lineEndOffset, textModel);
         if (!result) {
             return undefined;
         }
@@ -625,8 +713,7 @@ TreeSitterTokenizationSupport = ( __decorate([
     ( __param(4, ITreeSitterParserService)),
     ( __param(5, IWorkbenchThemeService)),
     ( __param(6, ITreeSitterTokenizationStoreService)),
-    ( __param(7, ICodeEditorService)),
-    ( __param(8, IInstantiationService))
+    ( __param(7, IInstantiationService))
 ], TreeSitterTokenizationSupport));
 
-export { TreeSitterTokenizationFeature, TreeSitterTokenizationSupport };
+export { TREESITTER_BASE_SCOPES, TreeSitterTokenizationFeature, TreeSitterTokenizationSupport };