monaco-editor11 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/css.worker-C7FogG4G.js +93 -0
- package/dist/assets/editor.worker-iXcRX1Tq.js +26 -0
- package/dist/assets/html.worker-C8VxctEJ.js +470 -0
- package/dist/assets/json.worker-CMC9kgPL.js +58 -0
- package/dist/assets/ts.worker-CtTJ3hNN.js +67731 -0
- package/dist/index.d.ts +0 -6
- package/dist/monaco-editor11.es.js +49 -78
- package/dist/monaco-editor11.umd.js +1 -50
- package/dist/monaco.d.ts +8 -0
- package/dist/workers/common/initialize.js +16 -0
- package/dist/workers/common/workers.js +141 -0
- package/dist/workers/editor/common/abstractSyntaxTokenBackend.js +128 -0
- package/dist/workers/editor/common/abstractText.js +89 -0
- package/dist/workers/editor/common/ast.js +485 -0
- package/dist/workers/editor/common/autoIndent.js +390 -0
- package/dist/workers/editor/common/beforeEditPositionMapper.js +110 -0
- package/dist/workers/editor/common/bracketPairsImpl.js +717 -0
- package/dist/workers/editor/common/bracketPairsTree.js +343 -0
- package/dist/workers/editor/common/brackets.js +108 -0
- package/dist/workers/editor/common/characterClassifier.js +59 -0
- package/dist/workers/editor/common/characterPair.js +40 -0
- package/dist/workers/editor/common/colorizedBracketPairsDecorationProvider.js +97 -0
- package/dist/workers/editor/common/columnRange.js +35 -0
- package/dist/workers/editor/common/combineTextEditInfos.js +124 -0
- package/dist/workers/editor/common/common.js +20 -0
- package/dist/workers/editor/common/computeMovedLines.js +249 -0
- package/dist/workers/editor/common/concat23Trees.js +192 -0
- package/dist/workers/editor/common/contiguousMultilineTokens.js +32 -0
- package/dist/workers/editor/common/contiguousMultilineTokensBuilder.js +23 -0
- package/dist/workers/editor/common/contiguousTokensEditing.js +128 -0
- package/dist/workers/editor/common/contiguousTokensStore.js +207 -0
- package/dist/workers/editor/common/coordinatesConverter.js +51 -0
- package/dist/workers/editor/common/cursor.js +899 -0
- package/dist/workers/editor/common/cursorAtomicMoveOperations.js +145 -0
- package/dist/workers/editor/common/cursorCollection.js +194 -0
- package/dist/workers/editor/common/cursorColumnSelection.js +93 -0
- package/dist/workers/editor/common/cursorColumns.js +112 -0
- package/dist/workers/editor/common/cursorCommon.js +250 -0
- package/dist/workers/editor/common/cursorContext.js +15 -0
- package/dist/workers/editor/common/cursorDeleteOperations.js +231 -0
- package/dist/workers/editor/common/cursorMoveCommands.js +676 -0
- package/dist/workers/editor/common/cursorMoveOperations.js +290 -0
- package/dist/workers/editor/common/cursorTypeEditOperations.js +968 -0
- package/dist/workers/editor/common/cursorTypeOperations.js +173 -0
- package/dist/workers/editor/common/cursorUtils.js +75 -0
- package/dist/workers/editor/common/cursorWordOperations.js +720 -0
- package/dist/workers/editor/common/defaultDocumentColorsComputer.js +138 -0
- package/dist/workers/editor/common/defaultLinesDiffComputer.js +188 -0
- package/dist/workers/editor/common/diffAlgorithm.js +139 -0
- package/dist/workers/editor/common/diffEditor.js +38 -0
- package/dist/workers/editor/common/dynamicProgrammingDiffing.js +101 -0
- package/dist/workers/editor/common/edit.js +183 -0
- package/dist/workers/editor/common/editOperation.js +36 -0
- package/dist/workers/editor/common/editStack.js +363 -0
- package/dist/workers/editor/common/editorAction.js +26 -0
- package/dist/workers/editor/common/editorBaseApi.js +43 -0
- package/dist/workers/editor/common/editorColorRegistry.js +102 -0
- package/dist/workers/editor/common/editorCommon.js +13 -0
- package/dist/workers/editor/common/editorConfigurationSchema.js +338 -0
- package/dist/workers/editor/common/editorContextKeys.js +84 -0
- package/dist/workers/editor/common/editorFeatures.js +17 -0
- package/dist/workers/editor/common/editorOptions.js +3440 -0
- package/dist/workers/editor/common/editorTheme.js +23 -0
- package/dist/workers/editor/common/editorWebWorker.js +299 -0
- package/dist/workers/editor/common/editorWorker.js +9 -0
- package/dist/workers/editor/common/editorWorkerHost.js +15 -0
- package/dist/workers/editor/common/editorZoom.js +26 -0
- package/dist/workers/editor/common/electricCharacter.js +55 -0
- package/dist/workers/editor/common/encodedTokenAttributes.js +79 -0
- package/dist/workers/editor/common/enterAction.js +53 -0
- package/dist/workers/editor/common/eolCounter.js +44 -0
- package/dist/workers/editor/common/findSectionHeaders.js +128 -0
- package/dist/workers/editor/common/fixBrackets.js +67 -0
- package/dist/workers/editor/common/fixedArray.js +70 -0
- package/dist/workers/editor/common/fontInfo.js +172 -0
- package/dist/workers/editor/common/fontInfoFromSettings.js +29 -0
- package/dist/workers/editor/common/getIconClasses.js +106 -0
- package/dist/workers/editor/common/getPositionOffsetTransformerFromTextModel.js +24 -0
- package/dist/workers/editor/common/glyphLanesModel.js +61 -0
- package/dist/workers/editor/common/guidesTextModelPart.js +405 -0
- package/dist/workers/editor/common/heuristicSequenceOptimizations.js +374 -0
- package/dist/workers/editor/common/indentRules.js +63 -0
- package/dist/workers/editor/common/indentation.js +39 -0
- package/dist/workers/editor/common/indentationGuesser.js +178 -0
- package/dist/workers/editor/common/indentationLineProcessor.js +193 -0
- package/dist/workers/editor/common/inlineDecorations.js +26 -0
- package/dist/workers/editor/common/inplaceReplaceSupport.js +87 -0
- package/dist/workers/editor/common/inputMode.js +22 -0
- package/dist/workers/editor/common/intervalTree.js +1002 -0
- package/dist/workers/editor/common/language.js +9 -0
- package/dist/workers/editor/common/languageBracketsConfiguration.js +133 -0
- package/dist/workers/editor/common/languageConfiguration.js +138 -0
- package/dist/workers/editor/common/languageConfigurationRegistry.js +361 -0
- package/dist/workers/editor/common/languageFeatureDebounce.js +137 -0
- package/dist/workers/editor/common/languageFeatureRegistry.js +180 -0
- package/dist/workers/editor/common/languageFeatures.js +9 -0
- package/dist/workers/editor/common/languageFeaturesService.js +47 -0
- package/dist/workers/editor/common/languageSelector.js +112 -0
- package/dist/workers/editor/common/languageService.js +92 -0
- package/dist/workers/editor/common/languages.js +522 -0
- package/dist/workers/editor/common/languagesAssociations.js +193 -0
- package/dist/workers/editor/common/languagesRegistry.js +237 -0
- package/dist/workers/editor/common/legacyLinesDiffComputer.js +468 -0
- package/dist/workers/editor/common/length.js +129 -0
- package/dist/workers/editor/common/lineDecorations.js +208 -0
- package/dist/workers/editor/common/lineEdit.js +75 -0
- package/dist/workers/editor/common/lineHeights.js +370 -0
- package/dist/workers/editor/common/linePart.js +25 -0
- package/dist/workers/editor/common/lineRange.js +312 -0
- package/dist/workers/editor/common/lineSequence.js +36 -0
- package/dist/workers/editor/common/lineTokens.js +405 -0
- package/dist/workers/editor/common/linesDiffComputer.js +29 -0
- package/dist/workers/editor/common/linesDiffComputers.js +13 -0
- package/dist/workers/editor/common/linesLayout.js +765 -0
- package/dist/workers/editor/common/linesSliceCharSequence.js +205 -0
- package/dist/workers/editor/common/linkComputer.js +269 -0
- package/dist/workers/editor/common/markerDecorations.js +9 -0
- package/dist/workers/editor/common/markerDecorationsService.js +248 -0
- package/dist/workers/editor/common/minimapTokensColorTracker.js +58 -0
- package/dist/workers/editor/common/mirrorTextModel.js +117 -0
- package/dist/workers/editor/common/model.js +9 -0
- package/dist/workers/editor/common/modelLineProjection.js +350 -0
- package/dist/workers/editor/common/modelLineProjectionData.js +297 -0
- package/dist/workers/editor/common/modelService.js +413 -0
- package/dist/workers/editor/common/modesRegistry.js +75 -0
- package/dist/workers/editor/common/monospaceLineBreaksComputer.js +473 -0
- package/dist/workers/editor/common/myersDiffAlgorithm.js +159 -0
- package/dist/workers/editor/common/nodeReader.js +127 -0
- package/dist/workers/editor/common/nullTokenize.js +29 -0
- package/dist/workers/editor/common/offsetRange.js +225 -0
- package/dist/workers/editor/common/onEnter.js +109 -0
- package/dist/workers/editor/common/oneCursor.js +117 -0
- package/dist/workers/editor/common/overviewZoneManager.js +176 -0
- package/dist/workers/editor/common/parser.js +121 -0
- package/dist/workers/editor/common/pieceTreeBase.js +1473 -0
- package/dist/workers/editor/common/pieceTreeTextBuffer.js +461 -0
- package/dist/workers/editor/common/pieceTreeTextBufferBuilder.js +140 -0
- package/dist/workers/editor/common/point.js +50 -0
- package/dist/workers/editor/common/position.js +142 -0
- package/dist/workers/editor/common/positionToOffset.js +17 -0
- package/dist/workers/editor/common/positionToOffsetImpl.js +98 -0
- package/dist/workers/editor/common/prefixSumComputer.js +226 -0
- package/dist/workers/editor/common/range.js +421 -0
- package/dist/workers/editor/common/rangeMapping.js +229 -0
- package/dist/workers/editor/common/rangeSingleLine.js +17 -0
- package/dist/workers/editor/common/rbTreeBase.js +362 -0
- package/dist/workers/editor/common/rect.js +163 -0
- package/dist/workers/editor/common/replaceCommand.js +158 -0
- package/dist/workers/editor/common/resolverService.js +5 -0
- package/dist/workers/editor/common/rgba.js +35 -0
- package/dist/workers/editor/common/richEditBrackets.js +356 -0
- package/dist/workers/editor/common/selection.js +145 -0
- package/dist/workers/editor/common/semanticTokensDto.js +82 -0
- package/dist/workers/editor/common/semanticTokensProviderStyling.js +263 -0
- package/dist/workers/editor/common/semanticTokensStyling.js +9 -0
- package/dist/workers/editor/common/semanticTokensStylingService.js +47 -0
- package/dist/workers/editor/common/shiftCommand.js +241 -0
- package/dist/workers/editor/common/smallImmutableSet.js +108 -0
- package/dist/workers/editor/common/sparseMultilineTokens.js +548 -0
- package/dist/workers/editor/common/sparseTokensStore.js +210 -0
- package/dist/workers/editor/common/standaloneEnums.js +1017 -0
- package/dist/workers/editor/common/standaloneStrings.js +42 -0
- package/dist/workers/editor/common/stringBuilder.js +122 -0
- package/dist/workers/editor/common/stringEdit.js +165 -0
- package/dist/workers/editor/common/supports.js +58 -0
- package/dist/workers/editor/common/surroundSelectionCommand.js +44 -0
- package/dist/workers/editor/common/textChange.js +248 -0
- package/dist/workers/editor/common/textEdit.js +269 -0
- package/dist/workers/editor/common/textLength.js +87 -0
- package/dist/workers/editor/common/textModel.js +2031 -0
- package/dist/workers/editor/common/textModelBracketPairs.js +45 -0
- package/dist/workers/editor/common/textModelDefaults.js +18 -0
- package/dist/workers/editor/common/textModelEditSource.js +166 -0
- package/dist/workers/editor/common/textModelEvents.js +216 -0
- package/dist/workers/editor/common/textModelGuides.js +40 -0
- package/dist/workers/editor/common/textModelPart.js +23 -0
- package/dist/workers/editor/common/textModelSearch.js +455 -0
- package/dist/workers/editor/common/textModelStringEdit.js +11 -0
- package/dist/workers/editor/common/textModelSync.impl.js +307 -0
- package/dist/workers/editor/common/textModelText.js +26 -0
- package/dist/workers/editor/common/textModelTokens.js +436 -0
- package/dist/workers/editor/common/textResourceConfiguration.js +6 -0
- package/dist/workers/editor/common/textToHtmlTokenizer.js +139 -0
- package/dist/workers/editor/common/tokenStore.js +407 -0
- package/dist/workers/editor/common/tokenWithTextArray.js +73 -0
- package/dist/workers/editor/common/tokenization.js +287 -0
- package/dist/workers/editor/common/tokenizationRegistry.js +123 -0
- package/dist/workers/editor/common/tokenizationTextModelPart.js +275 -0
- package/dist/workers/editor/common/tokenizer.js +301 -0
- package/dist/workers/editor/common/tokenizerSyntaxTokenBackend.js +261 -0
- package/dist/workers/editor/common/treeSitterLibraryService.js +9 -0
- package/dist/workers/editor/common/treeSitterSyntaxTokenBackend.js +167 -0
- package/dist/workers/editor/common/treeSitterThemeService.js +9 -0
- package/dist/workers/editor/common/treeSitterTokenizationImpl.js +713 -0
- package/dist/workers/editor/common/treeSitterTree.js +395 -0
- package/dist/workers/editor/common/treeViewsDnd.js +24 -0
- package/dist/workers/editor/common/treeViewsDndService.js +12 -0
- package/dist/workers/editor/common/trimTrailingWhitespaceCommand.js +98 -0
- package/dist/workers/editor/common/unicodeTextModelHighlighter.js +188 -0
- package/dist/workers/editor/common/utils.js +62 -0
- package/dist/workers/editor/common/viewContext.js +22 -0
- package/dist/workers/editor/common/viewEventHandler.js +186 -0
- package/dist/workers/editor/common/viewEvents.js +180 -0
- package/dist/workers/editor/common/viewLayout.js +368 -0
- package/dist/workers/editor/common/viewLineRenderer.js +948 -0
- package/dist/workers/editor/common/viewLinesViewportData.js +30 -0
- package/dist/workers/editor/common/viewModel.js +98 -0
- package/dist/workers/editor/common/viewModelDecoration.js +55 -0
- package/dist/workers/editor/common/viewModelDecorations.js +132 -0
- package/dist/workers/editor/common/viewModelEventDispatcher.js +398 -0
- package/dist/workers/editor/common/viewModelImpl.js +1163 -0
- package/dist/workers/editor/common/viewModelLines.js +938 -0
- package/dist/workers/editor/common/wordCharacterClassifier.js +87 -0
- package/dist/workers/editor/common/wordHelper.js +127 -0
- package/dist/workers/editor/editor.worker.js +11 -0
- package/dist/workers/language/css.worker.js +8 -0
- package/dist/workers/language/cssMode.js +198 -0
- package/dist/workers/language/cssWorker.js +183 -0
- package/dist/workers/language/html.worker.js +8 -0
- package/dist/workers/language/htmlMode.js +213 -0
- package/dist/workers/language/htmlWorker.js +126 -0
- package/dist/workers/language/json.worker.js +8 -0
- package/dist/workers/language/jsonMode.js +224 -0
- package/dist/workers/language/jsonWorker.js +187 -0
- package/dist/workers/language/languageFeatures.js +1009 -0
- package/dist/workers/language/lib.index.js +103 -0
- package/dist/workers/language/lib.js +1107 -0
- package/dist/workers/language/lspLanguageFeatures.js +716 -0
- package/dist/workers/language/monaco.contribution.js +144 -0
- package/dist/workers/language/tokenization.js +189 -0
- package/dist/workers/language/ts.worker.js +14 -0
- package/dist/workers/language/tsMode.js +212 -0
- package/dist/workers/language/tsWorker.js +352 -0
- package/dist/workers/language/typescriptServices.js +210154 -0
- package/dist/workers/language/typescriptServicesMetadata.js +3 -0
- package/dist/workers/language/workerManager.js +65 -0
- package/package.json +3 -2
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import { NotSupportedError } from '../../../../../base/common/errors.js';
|
|
2
|
+
import { TokenMetadata } from '../../../encodedTokenAttributes.js';
|
|
3
|
+
import { TextAstNode } from './ast.js';
|
|
4
|
+
import { lengthZero, toLength, lengthAdd, lengthToObj, lengthGetColumnCountIfZeroLineCount, lengthDiff } from './length.js';
|
|
5
|
+
import { SmallImmutableSet } from './smallImmutableSet.js';
|
|
6
|
+
|
|
7
|
+
/*---------------------------------------------------------------------------------------------
|
|
8
|
+
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
9
|
+
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
10
|
+
*--------------------------------------------------------------------------------------------*/
|
|
11
|
+
class Token {
|
|
12
|
+
constructor(length, kind,
|
|
13
|
+
/**
|
|
14
|
+
* If this token is an opening bracket, this is the id of the opening bracket.
|
|
15
|
+
* If this token is a closing bracket, this is the id of the first opening bracket that is closed by this bracket.
|
|
16
|
+
* Otherwise, it is -1.
|
|
17
|
+
*/
|
|
18
|
+
bracketId,
|
|
19
|
+
/**
|
|
20
|
+
* If this token is an opening bracket, this just contains `bracketId`.
|
|
21
|
+
* If this token is a closing bracket, this lists all opening bracket ids, that it closes.
|
|
22
|
+
* Otherwise, it is empty.
|
|
23
|
+
*/
|
|
24
|
+
bracketIds, astNode) {
|
|
25
|
+
this.length = length;
|
|
26
|
+
this.kind = kind;
|
|
27
|
+
this.bracketId = bracketId;
|
|
28
|
+
this.bracketIds = bracketIds;
|
|
29
|
+
this.astNode = astNode;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
class TextBufferTokenizer {
|
|
33
|
+
constructor(textModel, bracketTokens) {
|
|
34
|
+
this.textModel = textModel;
|
|
35
|
+
this.bracketTokens = bracketTokens;
|
|
36
|
+
this.reader = new NonPeekableTextBufferTokenizer(this.textModel, this.bracketTokens);
|
|
37
|
+
this._offset = lengthZero;
|
|
38
|
+
this.didPeek = false;
|
|
39
|
+
this.peeked = null;
|
|
40
|
+
this.textBufferLineCount = textModel.getLineCount();
|
|
41
|
+
this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
|
|
42
|
+
}
|
|
43
|
+
get offset() {
|
|
44
|
+
return this._offset;
|
|
45
|
+
}
|
|
46
|
+
get length() {
|
|
47
|
+
return toLength(this.textBufferLineCount - 1, this.textBufferLastLineLength);
|
|
48
|
+
}
|
|
49
|
+
skip(length) {
|
|
50
|
+
this.didPeek = false;
|
|
51
|
+
this._offset = lengthAdd(this._offset, length);
|
|
52
|
+
const obj = lengthToObj(this._offset);
|
|
53
|
+
this.reader.setPosition(obj.lineCount, obj.columnCount);
|
|
54
|
+
}
|
|
55
|
+
read() {
|
|
56
|
+
let token;
|
|
57
|
+
if (this.peeked) {
|
|
58
|
+
this.didPeek = false;
|
|
59
|
+
token = this.peeked;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
token = this.reader.read();
|
|
63
|
+
}
|
|
64
|
+
if (token) {
|
|
65
|
+
this._offset = lengthAdd(this._offset, token.length);
|
|
66
|
+
}
|
|
67
|
+
return token;
|
|
68
|
+
}
|
|
69
|
+
peek() {
|
|
70
|
+
if (!this.didPeek) {
|
|
71
|
+
this.peeked = this.reader.read();
|
|
72
|
+
this.didPeek = true;
|
|
73
|
+
}
|
|
74
|
+
return this.peeked;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Does not support peek.
|
|
79
|
+
*/
|
|
80
|
+
class NonPeekableTextBufferTokenizer {
|
|
81
|
+
constructor(textModel, bracketTokens) {
|
|
82
|
+
this.textModel = textModel;
|
|
83
|
+
this.bracketTokens = bracketTokens;
|
|
84
|
+
this.lineIdx = 0;
|
|
85
|
+
this.line = null;
|
|
86
|
+
this.lineCharOffset = 0;
|
|
87
|
+
this.lineTokens = null;
|
|
88
|
+
this.lineTokenOffset = 0;
|
|
89
|
+
/** Must be a zero line token. The end of the document cannot be peeked. */
|
|
90
|
+
this.peekedToken = null;
|
|
91
|
+
this.textBufferLineCount = textModel.getLineCount();
|
|
92
|
+
this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
|
|
93
|
+
}
|
|
94
|
+
setPosition(lineIdx, column) {
|
|
95
|
+
// We must not jump into a token!
|
|
96
|
+
if (lineIdx === this.lineIdx) {
|
|
97
|
+
this.lineCharOffset = column;
|
|
98
|
+
if (this.line !== null) {
|
|
99
|
+
this.lineTokenOffset = this.lineCharOffset === 0 ? 0 : this.lineTokens.findTokenIndexAtOffset(this.lineCharOffset);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
this.lineIdx = lineIdx;
|
|
104
|
+
this.lineCharOffset = column;
|
|
105
|
+
this.line = null;
|
|
106
|
+
}
|
|
107
|
+
this.peekedToken = null;
|
|
108
|
+
}
|
|
109
|
+
read() {
|
|
110
|
+
if (this.peekedToken) {
|
|
111
|
+
const token = this.peekedToken;
|
|
112
|
+
this.peekedToken = null;
|
|
113
|
+
this.lineCharOffset += lengthGetColumnCountIfZeroLineCount(token.length);
|
|
114
|
+
return token;
|
|
115
|
+
}
|
|
116
|
+
if (this.lineIdx > this.textBufferLineCount - 1 || (this.lineIdx === this.textBufferLineCount - 1 && this.lineCharOffset >= this.textBufferLastLineLength)) {
|
|
117
|
+
// We are after the end
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
if (this.line === null) {
|
|
121
|
+
this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
|
|
122
|
+
this.line = this.lineTokens.getLineContent();
|
|
123
|
+
this.lineTokenOffset = this.lineCharOffset === 0 ? 0 : this.lineTokens.findTokenIndexAtOffset(this.lineCharOffset);
|
|
124
|
+
}
|
|
125
|
+
const startLineIdx = this.lineIdx;
|
|
126
|
+
const startLineCharOffset = this.lineCharOffset;
|
|
127
|
+
// limits the length of text tokens.
|
|
128
|
+
// If text tokens get too long, incremental updates will be slow
|
|
129
|
+
let lengthHeuristic = 0;
|
|
130
|
+
while (true) {
|
|
131
|
+
const lineTokens = this.lineTokens;
|
|
132
|
+
const tokenCount = lineTokens.getCount();
|
|
133
|
+
let peekedBracketToken = null;
|
|
134
|
+
if (this.lineTokenOffset < tokenCount) {
|
|
135
|
+
const tokenMetadata = lineTokens.getMetadata(this.lineTokenOffset);
|
|
136
|
+
while (this.lineTokenOffset + 1 < tokenCount && tokenMetadata === lineTokens.getMetadata(this.lineTokenOffset + 1)) {
|
|
137
|
+
// Skip tokens that are identical.
|
|
138
|
+
// Sometimes, (bracket) identifiers are split up into multiple tokens.
|
|
139
|
+
this.lineTokenOffset++;
|
|
140
|
+
}
|
|
141
|
+
const isOther = TokenMetadata.getTokenType(tokenMetadata) === 0 /* StandardTokenType.Other */;
|
|
142
|
+
const containsBracketType = TokenMetadata.containsBalancedBrackets(tokenMetadata);
|
|
143
|
+
const endOffset = lineTokens.getEndOffset(this.lineTokenOffset);
|
|
144
|
+
// Is there a bracket token next? Only consume text.
|
|
145
|
+
if (containsBracketType && isOther && this.lineCharOffset < endOffset) {
|
|
146
|
+
const languageId = lineTokens.getLanguageId(this.lineTokenOffset);
|
|
147
|
+
const text = this.line.substring(this.lineCharOffset, endOffset);
|
|
148
|
+
const brackets = this.bracketTokens.getSingleLanguageBracketTokens(languageId);
|
|
149
|
+
const regexp = brackets.regExpGlobal;
|
|
150
|
+
if (regexp) {
|
|
151
|
+
regexp.lastIndex = 0;
|
|
152
|
+
const match = regexp.exec(text);
|
|
153
|
+
if (match) {
|
|
154
|
+
peekedBracketToken = brackets.getToken(match[0]);
|
|
155
|
+
if (peekedBracketToken) {
|
|
156
|
+
// Consume leading text of the token
|
|
157
|
+
this.lineCharOffset += match.index;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
lengthHeuristic += endOffset - this.lineCharOffset;
|
|
163
|
+
if (peekedBracketToken) {
|
|
164
|
+
// Don't skip the entire token, as a single token could contain multiple brackets.
|
|
165
|
+
if (startLineIdx !== this.lineIdx || startLineCharOffset !== this.lineCharOffset) {
|
|
166
|
+
// There is text before the bracket
|
|
167
|
+
this.peekedToken = peekedBracketToken;
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
// Consume the peeked token
|
|
172
|
+
this.lineCharOffset += lengthGetColumnCountIfZeroLineCount(peekedBracketToken.length);
|
|
173
|
+
return peekedBracketToken;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
// Skip the entire token, as the token contains no brackets at all.
|
|
178
|
+
this.lineTokenOffset++;
|
|
179
|
+
this.lineCharOffset = endOffset;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
if (this.lineIdx === this.textBufferLineCount - 1) {
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
this.lineIdx++;
|
|
187
|
+
this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
|
|
188
|
+
this.lineTokenOffset = 0;
|
|
189
|
+
this.line = this.lineTokens.getLineContent();
|
|
190
|
+
this.lineCharOffset = 0;
|
|
191
|
+
lengthHeuristic += 33; // max 1000/33 = 30 lines
|
|
192
|
+
// This limits the amount of work to recompute min-indentation
|
|
193
|
+
if (lengthHeuristic > 1000) {
|
|
194
|
+
// only break (automatically) at the end of line.
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (lengthHeuristic > 1500) {
|
|
199
|
+
// Eventually break regardless of the line length so that
|
|
200
|
+
// very long lines do not cause bad performance.
|
|
201
|
+
// This effective limits max indentation to 500, as
|
|
202
|
+
// indentation is not computed across multiple text nodes.
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// If a token contains some proper indentation, it also contains \n{INDENTATION+}(?!{INDENTATION}),
|
|
207
|
+
// unless the line is too long.
|
|
208
|
+
// Thus, the min indentation of the document is the minimum min indentation of every text node.
|
|
209
|
+
const length = lengthDiff(startLineIdx, startLineCharOffset, this.lineIdx, this.lineCharOffset);
|
|
210
|
+
return new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
class FastTokenizer {
|
|
214
|
+
constructor(text, brackets) {
|
|
215
|
+
this.text = text;
|
|
216
|
+
this._offset = lengthZero;
|
|
217
|
+
this.idx = 0;
|
|
218
|
+
const regExpStr = brackets.getRegExpStr();
|
|
219
|
+
const regexp = regExpStr ? new RegExp(regExpStr + '|\n', 'gi') : null;
|
|
220
|
+
const tokens = [];
|
|
221
|
+
let match;
|
|
222
|
+
let curLineCount = 0;
|
|
223
|
+
let lastLineBreakOffset = 0;
|
|
224
|
+
let lastTokenEndOffset = 0;
|
|
225
|
+
let lastTokenEndLine = 0;
|
|
226
|
+
const smallTextTokens0Line = [];
|
|
227
|
+
for (let i = 0; i < 60; i++) {
|
|
228
|
+
smallTextTokens0Line.push(new Token(toLength(0, i), 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(0, i))));
|
|
229
|
+
}
|
|
230
|
+
const smallTextTokens1Line = [];
|
|
231
|
+
for (let i = 0; i < 60; i++) {
|
|
232
|
+
smallTextTokens1Line.push(new Token(toLength(1, i), 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(1, i))));
|
|
233
|
+
}
|
|
234
|
+
if (regexp) {
|
|
235
|
+
regexp.lastIndex = 0;
|
|
236
|
+
// If a token contains indentation, it also contains \n{INDENTATION+}(?!{INDENTATION})
|
|
237
|
+
while ((match = regexp.exec(text)) !== null) {
|
|
238
|
+
const curOffset = match.index;
|
|
239
|
+
const value = match[0];
|
|
240
|
+
if (value === '\n') {
|
|
241
|
+
curLineCount++;
|
|
242
|
+
lastLineBreakOffset = curOffset + 1;
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
if (lastTokenEndOffset !== curOffset) {
|
|
246
|
+
let token;
|
|
247
|
+
if (lastTokenEndLine === curLineCount) {
|
|
248
|
+
const colCount = curOffset - lastTokenEndOffset;
|
|
249
|
+
if (colCount < smallTextTokens0Line.length) {
|
|
250
|
+
token = smallTextTokens0Line[colCount];
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
const length = toLength(0, colCount);
|
|
254
|
+
token = new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
else {
|
|
258
|
+
const lineCount = curLineCount - lastTokenEndLine;
|
|
259
|
+
const colCount = curOffset - lastLineBreakOffset;
|
|
260
|
+
if (lineCount === 1 && colCount < smallTextTokens1Line.length) {
|
|
261
|
+
token = smallTextTokens1Line[colCount];
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
const length = toLength(lineCount, colCount);
|
|
265
|
+
token = new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
tokens.push(token);
|
|
269
|
+
}
|
|
270
|
+
// value is matched by regexp, so the token must exist
|
|
271
|
+
tokens.push(brackets.getToken(value));
|
|
272
|
+
lastTokenEndOffset = curOffset + value.length;
|
|
273
|
+
lastTokenEndLine = curLineCount;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const offset = text.length;
|
|
278
|
+
if (lastTokenEndOffset !== offset) {
|
|
279
|
+
const length = (lastTokenEndLine === curLineCount)
|
|
280
|
+
? toLength(0, offset - lastTokenEndOffset)
|
|
281
|
+
: toLength(curLineCount - lastTokenEndLine, offset - lastLineBreakOffset);
|
|
282
|
+
tokens.push(new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length)));
|
|
283
|
+
}
|
|
284
|
+
this.length = toLength(curLineCount, offset - lastLineBreakOffset);
|
|
285
|
+
this.tokens = tokens;
|
|
286
|
+
}
|
|
287
|
+
get offset() {
|
|
288
|
+
return this._offset;
|
|
289
|
+
}
|
|
290
|
+
read() {
|
|
291
|
+
return this.tokens[this.idx++] || null;
|
|
292
|
+
}
|
|
293
|
+
peek() {
|
|
294
|
+
return this.tokens[this.idx] || null;
|
|
295
|
+
}
|
|
296
|
+
skip(length) {
|
|
297
|
+
throw new NotSupportedError();
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export { FastTokenizer, TextBufferTokenizer, Token };
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { onUnexpectedError } from '../../../../base/common/errors.js';
|
|
2
|
+
import { Emitter } from '../../../../base/common/event.js';
|
|
3
|
+
import { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';
|
|
4
|
+
import { countEOL } from '../../core/misc/eolCounter.js';
|
|
5
|
+
import { Position } from '../../core/position.js';
|
|
6
|
+
import { LineRange } from '../../core/ranges/lineRange.js';
|
|
7
|
+
import { TokenizationRegistry } from '../../languages.js';
|
|
8
|
+
import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';
|
|
9
|
+
import { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';
|
|
10
|
+
import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from '../textModelTokens.js';
|
|
11
|
+
import { AbstractSyntaxTokenBackend, AttachedViewHandler } from './abstractSyntaxTokenBackend.js';
|
|
12
|
+
|
|
13
|
+
/*---------------------------------------------------------------------------------------------
|
|
14
|
+
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
15
|
+
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
16
|
+
*--------------------------------------------------------------------------------------------*/
|
|
17
|
+
/** For TextMate */
|
|
18
|
+
class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
|
|
19
|
+
constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
|
|
20
|
+
super(languageIdCodec, textModel);
|
|
21
|
+
this.getLanguageId = getLanguageId;
|
|
22
|
+
this._tokenizer = null;
|
|
23
|
+
this._backgroundTokenizationState = 1 /* BackgroundTokenizationState.InProgress */;
|
|
24
|
+
this._onDidChangeBackgroundTokenizationState = this._register(new Emitter());
|
|
25
|
+
this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
|
|
26
|
+
this._defaultBackgroundTokenizer = null;
|
|
27
|
+
this._backgroundTokenizer = this._register(new MutableDisposable());
|
|
28
|
+
this._tokens = new ContiguousTokensStore(this._languageIdCodec);
|
|
29
|
+
this._debugBackgroundTokenizer = this._register(new MutableDisposable());
|
|
30
|
+
this._attachedViewStates = this._register(new DisposableMap());
|
|
31
|
+
this._register(TokenizationRegistry.onDidChange((e) => {
|
|
32
|
+
const languageId = this.getLanguageId();
|
|
33
|
+
if (e.changedLanguages.indexOf(languageId) === -1) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
this.todo_resetTokenization();
|
|
37
|
+
}));
|
|
38
|
+
this.todo_resetTokenization();
|
|
39
|
+
this._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {
|
|
40
|
+
if (state) {
|
|
41
|
+
let existing = this._attachedViewStates.get(view);
|
|
42
|
+
if (!existing) {
|
|
43
|
+
existing = new AttachedViewHandler(() => this.refreshRanges(existing.lineRanges));
|
|
44
|
+
this._attachedViewStates.set(view, existing);
|
|
45
|
+
}
|
|
46
|
+
existing.handleStateChange(state);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
this._attachedViewStates.deleteAndDispose(view);
|
|
50
|
+
}
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
todo_resetTokenization(fireTokenChangeEvent = true) {
|
|
54
|
+
this._tokens.flush();
|
|
55
|
+
this._debugBackgroundTokens?.flush();
|
|
56
|
+
if (this._debugBackgroundStates) {
|
|
57
|
+
this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
|
|
58
|
+
}
|
|
59
|
+
if (fireTokenChangeEvent) {
|
|
60
|
+
this._onDidChangeTokens.fire({
|
|
61
|
+
semanticTokensApplied: false,
|
|
62
|
+
ranges: [
|
|
63
|
+
{
|
|
64
|
+
fromLineNumber: 1,
|
|
65
|
+
toLineNumber: this._textModel.getLineCount(),
|
|
66
|
+
},
|
|
67
|
+
],
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
const initializeTokenization = () => {
|
|
71
|
+
if (this._textModel.isTooLargeForTokenization()) {
|
|
72
|
+
return [null, null];
|
|
73
|
+
}
|
|
74
|
+
const tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());
|
|
75
|
+
if (!tokenizationSupport) {
|
|
76
|
+
return [null, null];
|
|
77
|
+
}
|
|
78
|
+
let initialState;
|
|
79
|
+
try {
|
|
80
|
+
initialState = tokenizationSupport.getInitialState();
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
onUnexpectedError(e);
|
|
84
|
+
return [null, null];
|
|
85
|
+
}
|
|
86
|
+
return [tokenizationSupport, initialState];
|
|
87
|
+
};
|
|
88
|
+
const [tokenizationSupport, initialState] = initializeTokenization();
|
|
89
|
+
if (tokenizationSupport && initialState) {
|
|
90
|
+
this._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
this._tokenizer = null;
|
|
94
|
+
}
|
|
95
|
+
this._backgroundTokenizer.clear();
|
|
96
|
+
this._defaultBackgroundTokenizer = null;
|
|
97
|
+
if (this._tokenizer) {
|
|
98
|
+
const b = {
|
|
99
|
+
setTokens: (tokens) => {
|
|
100
|
+
this.setTokens(tokens);
|
|
101
|
+
},
|
|
102
|
+
backgroundTokenizationFinished: () => {
|
|
103
|
+
if (this._backgroundTokenizationState === 2 /* BackgroundTokenizationState.Completed */) {
|
|
104
|
+
// We already did a full tokenization and don't go back to progressing.
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
const newState = 2 /* BackgroundTokenizationState.Completed */;
|
|
108
|
+
this._backgroundTokenizationState = newState;
|
|
109
|
+
this._onDidChangeBackgroundTokenizationState.fire();
|
|
110
|
+
},
|
|
111
|
+
setEndState: (lineNumber, state) => {
|
|
112
|
+
if (!this._tokenizer) {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
|
|
116
|
+
// Don't accept states for definitely valid states, the renderer is ahead of the worker!
|
|
117
|
+
if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
|
|
118
|
+
this._tokenizer?.store.setEndState(lineNumber, state);
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
|
|
123
|
+
this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
|
|
124
|
+
}
|
|
125
|
+
if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {
|
|
126
|
+
this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
|
|
127
|
+
new DefaultBackgroundTokenizer(this._tokenizer, b);
|
|
128
|
+
this._defaultBackgroundTokenizer.handleChanges();
|
|
129
|
+
}
|
|
130
|
+
if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
|
|
131
|
+
this._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);
|
|
132
|
+
this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
|
|
133
|
+
this._debugBackgroundTokenizer.clear();
|
|
134
|
+
this._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {
|
|
135
|
+
setTokens: (tokens) => {
|
|
136
|
+
this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
|
|
137
|
+
},
|
|
138
|
+
backgroundTokenizationFinished() {
|
|
139
|
+
// NO OP
|
|
140
|
+
},
|
|
141
|
+
setEndState: (lineNumber, state) => {
|
|
142
|
+
this._debugBackgroundStates?.setEndState(lineNumber, state);
|
|
143
|
+
},
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
this._debugBackgroundTokens = undefined;
|
|
148
|
+
this._debugBackgroundStates = undefined;
|
|
149
|
+
this._debugBackgroundTokenizer.value = undefined;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
this.refreshAllVisibleLineTokens();
|
|
153
|
+
}
|
|
154
|
+
handleDidChangeAttached() {
|
|
155
|
+
this._defaultBackgroundTokenizer?.handleChanges();
|
|
156
|
+
}
|
|
157
|
+
handleDidChangeContent(e) {
|
|
158
|
+
if (e.isFlush) {
|
|
159
|
+
// Don't fire the event, as the view might not have got the text change event yet
|
|
160
|
+
this.todo_resetTokenization(false);
|
|
161
|
+
}
|
|
162
|
+
else if (!e.isEolChange) { // We don't have to do anything on an EOL change
|
|
163
|
+
for (const c of e.changes) {
|
|
164
|
+
const [eolCount, firstLineLength] = countEOL(c.text);
|
|
165
|
+
this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
|
|
166
|
+
this._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);
|
|
167
|
+
}
|
|
168
|
+
this._debugBackgroundStates?.acceptChanges(e.changes);
|
|
169
|
+
if (this._tokenizer) {
|
|
170
|
+
this._tokenizer.store.acceptChanges(e.changes);
|
|
171
|
+
}
|
|
172
|
+
this._defaultBackgroundTokenizer?.handleChanges();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
setTokens(tokens) {
|
|
176
|
+
const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
|
|
177
|
+
if (changes.length > 0) {
|
|
178
|
+
this._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });
|
|
179
|
+
}
|
|
180
|
+
return { changes: changes };
|
|
181
|
+
}
|
|
182
|
+
refreshAllVisibleLineTokens() {
|
|
183
|
+
const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));
|
|
184
|
+
this.refreshRanges(ranges);
|
|
185
|
+
}
|
|
186
|
+
refreshRanges(ranges) {
|
|
187
|
+
for (const range of ranges) {
|
|
188
|
+
this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
refreshRange(startLineNumber, endLineNumber) {
|
|
192
|
+
if (!this._tokenizer) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
startLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));
|
|
196
|
+
endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
|
|
197
|
+
const builder = new ContiguousMultilineTokensBuilder();
|
|
198
|
+
const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
|
|
199
|
+
const changedTokens = this.setTokens(builder.finalize());
|
|
200
|
+
if (heuristicTokens) {
|
|
201
|
+
// We overrode tokens with heuristically computed ones.
|
|
202
|
+
// Because old states might get reused (thus stopping invalidation),
|
|
203
|
+
// we have to explicitly request the tokens for the changed ranges again.
|
|
204
|
+
for (const c of changedTokens.changes) {
|
|
205
|
+
this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
this._defaultBackgroundTokenizer?.checkFinished();
|
|
209
|
+
}
|
|
210
|
+
forceTokenization(lineNumber) {
|
|
211
|
+
const builder = new ContiguousMultilineTokensBuilder();
|
|
212
|
+
this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
|
|
213
|
+
this.setTokens(builder.finalize());
|
|
214
|
+
this._defaultBackgroundTokenizer?.checkFinished();
|
|
215
|
+
}
|
|
216
|
+
hasAccurateTokensForLine(lineNumber) {
|
|
217
|
+
if (!this._tokenizer) {
|
|
218
|
+
return true;
|
|
219
|
+
}
|
|
220
|
+
return this._tokenizer.hasAccurateTokensForLine(lineNumber);
|
|
221
|
+
}
|
|
222
|
+
isCheapToTokenize(lineNumber) {
|
|
223
|
+
if (!this._tokenizer) {
|
|
224
|
+
return true;
|
|
225
|
+
}
|
|
226
|
+
return this._tokenizer.isCheapToTokenize(lineNumber);
|
|
227
|
+
}
|
|
228
|
+
getLineTokens(lineNumber) {
|
|
229
|
+
const lineText = this._textModel.getLineContent(lineNumber);
|
|
230
|
+
const result = this._tokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
|
|
231
|
+
if (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {
|
|
232
|
+
if (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {
|
|
233
|
+
const backgroundResult = this._debugBackgroundTokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
|
|
234
|
+
if (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {
|
|
235
|
+
this._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
|
|
242
|
+
if (!this._tokenizer) {
|
|
243
|
+
return 0 /* StandardTokenType.Other */;
|
|
244
|
+
}
|
|
245
|
+
const position = this._textModel.validatePosition(new Position(lineNumber, column));
|
|
246
|
+
this.forceTokenization(position.lineNumber);
|
|
247
|
+
return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
|
|
248
|
+
}
|
|
249
|
+
tokenizeLinesAt(lineNumber, lines) {
|
|
250
|
+
if (!this._tokenizer) {
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
this.forceTokenization(lineNumber);
|
|
254
|
+
return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
|
|
255
|
+
}
|
|
256
|
+
get hasTokens() {
|
|
257
|
+
return this._tokens.hasTokens;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export { TokenizerSyntaxTokenBackend };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
|
2
|
+
|
|
3
|
+
/*---------------------------------------------------------------------------------------------
|
|
4
|
+
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
5
|
+
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
6
|
+
*--------------------------------------------------------------------------------------------*/
|
|
7
|
+
const ITreeSitterLibraryService = createDecorator('treeSitterLibraryService');
|
|
8
|
+
|
|
9
|
+
export { ITreeSitterLibraryService };
|