monaco-editor11 1.0.9 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/dist/index.d.ts +1 -1
  2. package/dist/monaco-editor11.es.js +12 -16
  3. package/dist/monaco-editor11.umd.js +1 -1
  4. package/package.json +2 -2
  5. package/dist/monaco.d.ts +0 -8
  6. package/dist/workers/common/initialize.js +0 -16
  7. package/dist/workers/common/workers.js +0 -141
  8. package/dist/workers/editor/common/abstractSyntaxTokenBackend.js +0 -128
  9. package/dist/workers/editor/common/abstractText.js +0 -89
  10. package/dist/workers/editor/common/ast.js +0 -485
  11. package/dist/workers/editor/common/autoIndent.js +0 -390
  12. package/dist/workers/editor/common/beforeEditPositionMapper.js +0 -110
  13. package/dist/workers/editor/common/bracketPairsImpl.js +0 -717
  14. package/dist/workers/editor/common/bracketPairsTree.js +0 -343
  15. package/dist/workers/editor/common/brackets.js +0 -108
  16. package/dist/workers/editor/common/characterClassifier.js +0 -59
  17. package/dist/workers/editor/common/characterPair.js +0 -40
  18. package/dist/workers/editor/common/colorizedBracketPairsDecorationProvider.js +0 -97
  19. package/dist/workers/editor/common/columnRange.js +0 -35
  20. package/dist/workers/editor/common/combineTextEditInfos.js +0 -124
  21. package/dist/workers/editor/common/common.js +0 -20
  22. package/dist/workers/editor/common/computeMovedLines.js +0 -249
  23. package/dist/workers/editor/common/concat23Trees.js +0 -192
  24. package/dist/workers/editor/common/contiguousMultilineTokens.js +0 -32
  25. package/dist/workers/editor/common/contiguousMultilineTokensBuilder.js +0 -23
  26. package/dist/workers/editor/common/contiguousTokensEditing.js +0 -128
  27. package/dist/workers/editor/common/contiguousTokensStore.js +0 -207
  28. package/dist/workers/editor/common/coordinatesConverter.js +0 -51
  29. package/dist/workers/editor/common/cursor.js +0 -899
  30. package/dist/workers/editor/common/cursorAtomicMoveOperations.js +0 -145
  31. package/dist/workers/editor/common/cursorCollection.js +0 -194
  32. package/dist/workers/editor/common/cursorColumnSelection.js +0 -93
  33. package/dist/workers/editor/common/cursorColumns.js +0 -112
  34. package/dist/workers/editor/common/cursorCommon.js +0 -250
  35. package/dist/workers/editor/common/cursorContext.js +0 -15
  36. package/dist/workers/editor/common/cursorDeleteOperations.js +0 -231
  37. package/dist/workers/editor/common/cursorMoveCommands.js +0 -676
  38. package/dist/workers/editor/common/cursorMoveOperations.js +0 -290
  39. package/dist/workers/editor/common/cursorTypeEditOperations.js +0 -968
  40. package/dist/workers/editor/common/cursorTypeOperations.js +0 -173
  41. package/dist/workers/editor/common/cursorUtils.js +0 -75
  42. package/dist/workers/editor/common/cursorWordOperations.js +0 -720
  43. package/dist/workers/editor/common/defaultDocumentColorsComputer.js +0 -138
  44. package/dist/workers/editor/common/defaultLinesDiffComputer.js +0 -188
  45. package/dist/workers/editor/common/diffAlgorithm.js +0 -139
  46. package/dist/workers/editor/common/diffEditor.js +0 -38
  47. package/dist/workers/editor/common/dynamicProgrammingDiffing.js +0 -101
  48. package/dist/workers/editor/common/edit.js +0 -183
  49. package/dist/workers/editor/common/editOperation.js +0 -36
  50. package/dist/workers/editor/common/editStack.js +0 -363
  51. package/dist/workers/editor/common/editorAction.js +0 -26
  52. package/dist/workers/editor/common/editorBaseApi.js +0 -43
  53. package/dist/workers/editor/common/editorColorRegistry.js +0 -102
  54. package/dist/workers/editor/common/editorCommon.js +0 -13
  55. package/dist/workers/editor/common/editorConfigurationSchema.js +0 -338
  56. package/dist/workers/editor/common/editorContextKeys.js +0 -84
  57. package/dist/workers/editor/common/editorFeatures.js +0 -17
  58. package/dist/workers/editor/common/editorOptions.js +0 -3440
  59. package/dist/workers/editor/common/editorTheme.js +0 -23
  60. package/dist/workers/editor/common/editorWebWorker.js +0 -299
  61. package/dist/workers/editor/common/editorWorker.js +0 -9
  62. package/dist/workers/editor/common/editorWorkerHost.js +0 -15
  63. package/dist/workers/editor/common/editorZoom.js +0 -26
  64. package/dist/workers/editor/common/electricCharacter.js +0 -55
  65. package/dist/workers/editor/common/encodedTokenAttributes.js +0 -79
  66. package/dist/workers/editor/common/enterAction.js +0 -53
  67. package/dist/workers/editor/common/eolCounter.js +0 -44
  68. package/dist/workers/editor/common/findSectionHeaders.js +0 -128
  69. package/dist/workers/editor/common/fixBrackets.js +0 -67
  70. package/dist/workers/editor/common/fixedArray.js +0 -70
  71. package/dist/workers/editor/common/fontInfo.js +0 -172
  72. package/dist/workers/editor/common/fontInfoFromSettings.js +0 -29
  73. package/dist/workers/editor/common/getIconClasses.js +0 -106
  74. package/dist/workers/editor/common/getPositionOffsetTransformerFromTextModel.js +0 -24
  75. package/dist/workers/editor/common/glyphLanesModel.js +0 -61
  76. package/dist/workers/editor/common/guidesTextModelPart.js +0 -405
  77. package/dist/workers/editor/common/heuristicSequenceOptimizations.js +0 -374
  78. package/dist/workers/editor/common/indentRules.js +0 -63
  79. package/dist/workers/editor/common/indentation.js +0 -39
  80. package/dist/workers/editor/common/indentationGuesser.js +0 -178
  81. package/dist/workers/editor/common/indentationLineProcessor.js +0 -193
  82. package/dist/workers/editor/common/inlineDecorations.js +0 -26
  83. package/dist/workers/editor/common/inplaceReplaceSupport.js +0 -87
  84. package/dist/workers/editor/common/inputMode.js +0 -22
  85. package/dist/workers/editor/common/intervalTree.js +0 -1002
  86. package/dist/workers/editor/common/language.js +0 -9
  87. package/dist/workers/editor/common/languageBracketsConfiguration.js +0 -133
  88. package/dist/workers/editor/common/languageConfiguration.js +0 -138
  89. package/dist/workers/editor/common/languageConfigurationRegistry.js +0 -361
  90. package/dist/workers/editor/common/languageFeatureDebounce.js +0 -137
  91. package/dist/workers/editor/common/languageFeatureRegistry.js +0 -180
  92. package/dist/workers/editor/common/languageFeatures.js +0 -9
  93. package/dist/workers/editor/common/languageFeaturesService.js +0 -47
  94. package/dist/workers/editor/common/languageSelector.js +0 -112
  95. package/dist/workers/editor/common/languageService.js +0 -92
  96. package/dist/workers/editor/common/languages.js +0 -522
  97. package/dist/workers/editor/common/languagesAssociations.js +0 -193
  98. package/dist/workers/editor/common/languagesRegistry.js +0 -237
  99. package/dist/workers/editor/common/legacyLinesDiffComputer.js +0 -468
  100. package/dist/workers/editor/common/length.js +0 -129
  101. package/dist/workers/editor/common/lineDecorations.js +0 -208
  102. package/dist/workers/editor/common/lineEdit.js +0 -75
  103. package/dist/workers/editor/common/lineHeights.js +0 -370
  104. package/dist/workers/editor/common/linePart.js +0 -25
  105. package/dist/workers/editor/common/lineRange.js +0 -312
  106. package/dist/workers/editor/common/lineSequence.js +0 -36
  107. package/dist/workers/editor/common/lineTokens.js +0 -405
  108. package/dist/workers/editor/common/linesDiffComputer.js +0 -29
  109. package/dist/workers/editor/common/linesDiffComputers.js +0 -13
  110. package/dist/workers/editor/common/linesLayout.js +0 -765
  111. package/dist/workers/editor/common/linesSliceCharSequence.js +0 -205
  112. package/dist/workers/editor/common/linkComputer.js +0 -269
  113. package/dist/workers/editor/common/markerDecorations.js +0 -9
  114. package/dist/workers/editor/common/markerDecorationsService.js +0 -248
  115. package/dist/workers/editor/common/minimapTokensColorTracker.js +0 -58
  116. package/dist/workers/editor/common/mirrorTextModel.js +0 -117
  117. package/dist/workers/editor/common/model.js +0 -9
  118. package/dist/workers/editor/common/modelLineProjection.js +0 -350
  119. package/dist/workers/editor/common/modelLineProjectionData.js +0 -297
  120. package/dist/workers/editor/common/modelService.js +0 -413
  121. package/dist/workers/editor/common/modesRegistry.js +0 -75
  122. package/dist/workers/editor/common/monospaceLineBreaksComputer.js +0 -473
  123. package/dist/workers/editor/common/myersDiffAlgorithm.js +0 -159
  124. package/dist/workers/editor/common/nodeReader.js +0 -127
  125. package/dist/workers/editor/common/nullTokenize.js +0 -29
  126. package/dist/workers/editor/common/offsetRange.js +0 -225
  127. package/dist/workers/editor/common/onEnter.js +0 -109
  128. package/dist/workers/editor/common/oneCursor.js +0 -117
  129. package/dist/workers/editor/common/overviewZoneManager.js +0 -176
  130. package/dist/workers/editor/common/parser.js +0 -121
  131. package/dist/workers/editor/common/pieceTreeBase.js +0 -1473
  132. package/dist/workers/editor/common/pieceTreeTextBuffer.js +0 -461
  133. package/dist/workers/editor/common/pieceTreeTextBufferBuilder.js +0 -140
  134. package/dist/workers/editor/common/point.js +0 -50
  135. package/dist/workers/editor/common/position.js +0 -142
  136. package/dist/workers/editor/common/positionToOffset.js +0 -17
  137. package/dist/workers/editor/common/positionToOffsetImpl.js +0 -98
  138. package/dist/workers/editor/common/prefixSumComputer.js +0 -226
  139. package/dist/workers/editor/common/range.js +0 -421
  140. package/dist/workers/editor/common/rangeMapping.js +0 -229
  141. package/dist/workers/editor/common/rangeSingleLine.js +0 -17
  142. package/dist/workers/editor/common/rbTreeBase.js +0 -362
  143. package/dist/workers/editor/common/rect.js +0 -163
  144. package/dist/workers/editor/common/replaceCommand.js +0 -158
  145. package/dist/workers/editor/common/resolverService.js +0 -5
  146. package/dist/workers/editor/common/rgba.js +0 -35
  147. package/dist/workers/editor/common/richEditBrackets.js +0 -356
  148. package/dist/workers/editor/common/selection.js +0 -145
  149. package/dist/workers/editor/common/semanticTokensDto.js +0 -82
  150. package/dist/workers/editor/common/semanticTokensProviderStyling.js +0 -263
  151. package/dist/workers/editor/common/semanticTokensStyling.js +0 -9
  152. package/dist/workers/editor/common/semanticTokensStylingService.js +0 -47
  153. package/dist/workers/editor/common/shiftCommand.js +0 -241
  154. package/dist/workers/editor/common/smallImmutableSet.js +0 -108
  155. package/dist/workers/editor/common/sparseMultilineTokens.js +0 -548
  156. package/dist/workers/editor/common/sparseTokensStore.js +0 -210
  157. package/dist/workers/editor/common/standaloneEnums.js +0 -1017
  158. package/dist/workers/editor/common/standaloneStrings.js +0 -42
  159. package/dist/workers/editor/common/stringBuilder.js +0 -122
  160. package/dist/workers/editor/common/stringEdit.js +0 -165
  161. package/dist/workers/editor/common/supports.js +0 -58
  162. package/dist/workers/editor/common/surroundSelectionCommand.js +0 -44
  163. package/dist/workers/editor/common/textChange.js +0 -248
  164. package/dist/workers/editor/common/textEdit.js +0 -269
  165. package/dist/workers/editor/common/textLength.js +0 -87
  166. package/dist/workers/editor/common/textModel.js +0 -2031
  167. package/dist/workers/editor/common/textModelBracketPairs.js +0 -45
  168. package/dist/workers/editor/common/textModelDefaults.js +0 -18
  169. package/dist/workers/editor/common/textModelEditSource.js +0 -166
  170. package/dist/workers/editor/common/textModelEvents.js +0 -216
  171. package/dist/workers/editor/common/textModelGuides.js +0 -40
  172. package/dist/workers/editor/common/textModelPart.js +0 -23
  173. package/dist/workers/editor/common/textModelSearch.js +0 -455
  174. package/dist/workers/editor/common/textModelStringEdit.js +0 -11
  175. package/dist/workers/editor/common/textModelSync.impl.js +0 -307
  176. package/dist/workers/editor/common/textModelText.js +0 -26
  177. package/dist/workers/editor/common/textModelTokens.js +0 -436
  178. package/dist/workers/editor/common/textResourceConfiguration.js +0 -6
  179. package/dist/workers/editor/common/textToHtmlTokenizer.js +0 -139
  180. package/dist/workers/editor/common/tokenStore.js +0 -407
  181. package/dist/workers/editor/common/tokenWithTextArray.js +0 -73
  182. package/dist/workers/editor/common/tokenization.js +0 -287
  183. package/dist/workers/editor/common/tokenizationRegistry.js +0 -123
  184. package/dist/workers/editor/common/tokenizationTextModelPart.js +0 -275
  185. package/dist/workers/editor/common/tokenizer.js +0 -301
  186. package/dist/workers/editor/common/tokenizerSyntaxTokenBackend.js +0 -261
  187. package/dist/workers/editor/common/treeSitterLibraryService.js +0 -9
  188. package/dist/workers/editor/common/treeSitterSyntaxTokenBackend.js +0 -167
  189. package/dist/workers/editor/common/treeSitterThemeService.js +0 -9
  190. package/dist/workers/editor/common/treeSitterTokenizationImpl.js +0 -713
  191. package/dist/workers/editor/common/treeSitterTree.js +0 -395
  192. package/dist/workers/editor/common/treeViewsDnd.js +0 -24
  193. package/dist/workers/editor/common/treeViewsDndService.js +0 -12
  194. package/dist/workers/editor/common/trimTrailingWhitespaceCommand.js +0 -98
  195. package/dist/workers/editor/common/unicodeTextModelHighlighter.js +0 -188
  196. package/dist/workers/editor/common/utils.js +0 -62
  197. package/dist/workers/editor/common/viewContext.js +0 -22
  198. package/dist/workers/editor/common/viewEventHandler.js +0 -186
  199. package/dist/workers/editor/common/viewEvents.js +0 -180
  200. package/dist/workers/editor/common/viewLayout.js +0 -368
  201. package/dist/workers/editor/common/viewLineRenderer.js +0 -948
  202. package/dist/workers/editor/common/viewLinesViewportData.js +0 -30
  203. package/dist/workers/editor/common/viewModel.js +0 -98
  204. package/dist/workers/editor/common/viewModelDecoration.js +0 -55
  205. package/dist/workers/editor/common/viewModelDecorations.js +0 -132
  206. package/dist/workers/editor/common/viewModelEventDispatcher.js +0 -398
  207. package/dist/workers/editor/common/viewModelImpl.js +0 -1163
  208. package/dist/workers/editor/common/viewModelLines.js +0 -938
  209. package/dist/workers/editor/common/wordCharacterClassifier.js +0 -87
  210. package/dist/workers/editor/common/wordHelper.js +0 -127
  211. package/dist/workers/language/cssMode.js +0 -198
  212. package/dist/workers/language/cssWorker.js +0 -183
  213. package/dist/workers/language/htmlMode.js +0 -213
  214. package/dist/workers/language/htmlWorker.js +0 -126
  215. package/dist/workers/language/jsonMode.js +0 -224
  216. package/dist/workers/language/jsonWorker.js +0 -187
  217. package/dist/workers/language/languageFeatures.js +0 -1009
  218. package/dist/workers/language/lib.index.js +0 -103
  219. package/dist/workers/language/lib.js +0 -1107
  220. package/dist/workers/language/lspLanguageFeatures.js +0 -716
  221. package/dist/workers/language/monaco.contribution.js +0 -144
  222. package/dist/workers/language/tokenization.js +0 -189
  223. package/dist/workers/language/tsMode.js +0 -212
  224. package/dist/workers/language/tsWorker.js +0 -352
  225. package/dist/workers/language/typescriptServices.js +0 -210154
  226. package/dist/workers/language/typescriptServicesMetadata.js +0 -3
  227. package/dist/workers/language/workerManager.js +0 -65
  228. /package/dist/workers/{language/css.worker.js → css.worker.js} +0 -0
  229. /package/dist/workers/{editor/editor.worker.js → editor.worker.js} +0 -0
  230. /package/dist/workers/{language/html.worker.js → html.worker.js} +0 -0
  231. /package/dist/workers/{language/json.worker.js → json.worker.js} +0 -0
  232. /package/dist/workers/{language/ts.worker.js → ts.worker.js} +0 -0
@@ -1,301 +0,0 @@
1
- import { NotSupportedError } from '../../../../../base/common/errors.js';
2
- import { TokenMetadata } from '../../../encodedTokenAttributes.js';
3
- import { TextAstNode } from './ast.js';
4
- import { lengthZero, toLength, lengthAdd, lengthToObj, lengthGetColumnCountIfZeroLineCount, lengthDiff } from './length.js';
5
- import { SmallImmutableSet } from './smallImmutableSet.js';
6
-
7
- /*---------------------------------------------------------------------------------------------
8
- * Copyright (c) Microsoft Corporation. All rights reserved.
9
- * Licensed under the MIT License. See License.txt in the project root for license information.
10
- *--------------------------------------------------------------------------------------------*/
11
- class Token {
12
- constructor(length, kind,
13
- /**
14
- * If this token is an opening bracket, this is the id of the opening bracket.
15
- * If this token is a closing bracket, this is the id of the first opening bracket that is closed by this bracket.
16
- * Otherwise, it is -1.
17
- */
18
- bracketId,
19
- /**
20
- * If this token is an opening bracket, this just contains `bracketId`.
21
- * If this token is a closing bracket, this lists all opening bracket ids, that it closes.
22
- * Otherwise, it is empty.
23
- */
24
- bracketIds, astNode) {
25
- this.length = length;
26
- this.kind = kind;
27
- this.bracketId = bracketId;
28
- this.bracketIds = bracketIds;
29
- this.astNode = astNode;
30
- }
31
- }
32
- class TextBufferTokenizer {
33
- constructor(textModel, bracketTokens) {
34
- this.textModel = textModel;
35
- this.bracketTokens = bracketTokens;
36
- this.reader = new NonPeekableTextBufferTokenizer(this.textModel, this.bracketTokens);
37
- this._offset = lengthZero;
38
- this.didPeek = false;
39
- this.peeked = null;
40
- this.textBufferLineCount = textModel.getLineCount();
41
- this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
42
- }
43
- get offset() {
44
- return this._offset;
45
- }
46
- get length() {
47
- return toLength(this.textBufferLineCount - 1, this.textBufferLastLineLength);
48
- }
49
- skip(length) {
50
- this.didPeek = false;
51
- this._offset = lengthAdd(this._offset, length);
52
- const obj = lengthToObj(this._offset);
53
- this.reader.setPosition(obj.lineCount, obj.columnCount);
54
- }
55
- read() {
56
- let token;
57
- if (this.peeked) {
58
- this.didPeek = false;
59
- token = this.peeked;
60
- }
61
- else {
62
- token = this.reader.read();
63
- }
64
- if (token) {
65
- this._offset = lengthAdd(this._offset, token.length);
66
- }
67
- return token;
68
- }
69
- peek() {
70
- if (!this.didPeek) {
71
- this.peeked = this.reader.read();
72
- this.didPeek = true;
73
- }
74
- return this.peeked;
75
- }
76
- }
77
- /**
78
- * Does not support peek.
79
- */
80
- class NonPeekableTextBufferTokenizer {
81
- constructor(textModel, bracketTokens) {
82
- this.textModel = textModel;
83
- this.bracketTokens = bracketTokens;
84
- this.lineIdx = 0;
85
- this.line = null;
86
- this.lineCharOffset = 0;
87
- this.lineTokens = null;
88
- this.lineTokenOffset = 0;
89
- /** Must be a zero line token. The end of the document cannot be peeked. */
90
- this.peekedToken = null;
91
- this.textBufferLineCount = textModel.getLineCount();
92
- this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
93
- }
94
- setPosition(lineIdx, column) {
95
- // We must not jump into a token!
96
- if (lineIdx === this.lineIdx) {
97
- this.lineCharOffset = column;
98
- if (this.line !== null) {
99
- this.lineTokenOffset = this.lineCharOffset === 0 ? 0 : this.lineTokens.findTokenIndexAtOffset(this.lineCharOffset);
100
- }
101
- }
102
- else {
103
- this.lineIdx = lineIdx;
104
- this.lineCharOffset = column;
105
- this.line = null;
106
- }
107
- this.peekedToken = null;
108
- }
109
- read() {
110
- if (this.peekedToken) {
111
- const token = this.peekedToken;
112
- this.peekedToken = null;
113
- this.lineCharOffset += lengthGetColumnCountIfZeroLineCount(token.length);
114
- return token;
115
- }
116
- if (this.lineIdx > this.textBufferLineCount - 1 || (this.lineIdx === this.textBufferLineCount - 1 && this.lineCharOffset >= this.textBufferLastLineLength)) {
117
- // We are after the end
118
- return null;
119
- }
120
- if (this.line === null) {
121
- this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
122
- this.line = this.lineTokens.getLineContent();
123
- this.lineTokenOffset = this.lineCharOffset === 0 ? 0 : this.lineTokens.findTokenIndexAtOffset(this.lineCharOffset);
124
- }
125
- const startLineIdx = this.lineIdx;
126
- const startLineCharOffset = this.lineCharOffset;
127
- // limits the length of text tokens.
128
- // If text tokens get too long, incremental updates will be slow
129
- let lengthHeuristic = 0;
130
- while (true) {
131
- const lineTokens = this.lineTokens;
132
- const tokenCount = lineTokens.getCount();
133
- let peekedBracketToken = null;
134
- if (this.lineTokenOffset < tokenCount) {
135
- const tokenMetadata = lineTokens.getMetadata(this.lineTokenOffset);
136
- while (this.lineTokenOffset + 1 < tokenCount && tokenMetadata === lineTokens.getMetadata(this.lineTokenOffset + 1)) {
137
- // Skip tokens that are identical.
138
- // Sometimes, (bracket) identifiers are split up into multiple tokens.
139
- this.lineTokenOffset++;
140
- }
141
- const isOther = TokenMetadata.getTokenType(tokenMetadata) === 0 /* StandardTokenType.Other */;
142
- const containsBracketType = TokenMetadata.containsBalancedBrackets(tokenMetadata);
143
- const endOffset = lineTokens.getEndOffset(this.lineTokenOffset);
144
- // Is there a bracket token next? Only consume text.
145
- if (containsBracketType && isOther && this.lineCharOffset < endOffset) {
146
- const languageId = lineTokens.getLanguageId(this.lineTokenOffset);
147
- const text = this.line.substring(this.lineCharOffset, endOffset);
148
- const brackets = this.bracketTokens.getSingleLanguageBracketTokens(languageId);
149
- const regexp = brackets.regExpGlobal;
150
- if (regexp) {
151
- regexp.lastIndex = 0;
152
- const match = regexp.exec(text);
153
- if (match) {
154
- peekedBracketToken = brackets.getToken(match[0]);
155
- if (peekedBracketToken) {
156
- // Consume leading text of the token
157
- this.lineCharOffset += match.index;
158
- }
159
- }
160
- }
161
- }
162
- lengthHeuristic += endOffset - this.lineCharOffset;
163
- if (peekedBracketToken) {
164
- // Don't skip the entire token, as a single token could contain multiple brackets.
165
- if (startLineIdx !== this.lineIdx || startLineCharOffset !== this.lineCharOffset) {
166
- // There is text before the bracket
167
- this.peekedToken = peekedBracketToken;
168
- break;
169
- }
170
- else {
171
- // Consume the peeked token
172
- this.lineCharOffset += lengthGetColumnCountIfZeroLineCount(peekedBracketToken.length);
173
- return peekedBracketToken;
174
- }
175
- }
176
- else {
177
- // Skip the entire token, as the token contains no brackets at all.
178
- this.lineTokenOffset++;
179
- this.lineCharOffset = endOffset;
180
- }
181
- }
182
- else {
183
- if (this.lineIdx === this.textBufferLineCount - 1) {
184
- break;
185
- }
186
- this.lineIdx++;
187
- this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
188
- this.lineTokenOffset = 0;
189
- this.line = this.lineTokens.getLineContent();
190
- this.lineCharOffset = 0;
191
- lengthHeuristic += 33; // max 1000/33 = 30 lines
192
- // This limits the amount of work to recompute min-indentation
193
- if (lengthHeuristic > 1000) {
194
- // only break (automatically) at the end of line.
195
- break;
196
- }
197
- }
198
- if (lengthHeuristic > 1500) {
199
- // Eventually break regardless of the line length so that
200
- // very long lines do not cause bad performance.
201
- // This effective limits max indentation to 500, as
202
- // indentation is not computed across multiple text nodes.
203
- break;
204
- }
205
- }
206
- // If a token contains some proper indentation, it also contains \n{INDENTATION+}(?!{INDENTATION}),
207
- // unless the line is too long.
208
- // Thus, the min indentation of the document is the minimum min indentation of every text node.
209
- const length = lengthDiff(startLineIdx, startLineCharOffset, this.lineIdx, this.lineCharOffset);
210
- return new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
211
- }
212
- }
213
- class FastTokenizer {
214
- constructor(text, brackets) {
215
- this.text = text;
216
- this._offset = lengthZero;
217
- this.idx = 0;
218
- const regExpStr = brackets.getRegExpStr();
219
- const regexp = regExpStr ? new RegExp(regExpStr + '|\n', 'gi') : null;
220
- const tokens = [];
221
- let match;
222
- let curLineCount = 0;
223
- let lastLineBreakOffset = 0;
224
- let lastTokenEndOffset = 0;
225
- let lastTokenEndLine = 0;
226
- const smallTextTokens0Line = [];
227
- for (let i = 0; i < 60; i++) {
228
- smallTextTokens0Line.push(new Token(toLength(0, i), 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(0, i))));
229
- }
230
- const smallTextTokens1Line = [];
231
- for (let i = 0; i < 60; i++) {
232
- smallTextTokens1Line.push(new Token(toLength(1, i), 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(toLength(1, i))));
233
- }
234
- if (regexp) {
235
- regexp.lastIndex = 0;
236
- // If a token contains indentation, it also contains \n{INDENTATION+}(?!{INDENTATION})
237
- while ((match = regexp.exec(text)) !== null) {
238
- const curOffset = match.index;
239
- const value = match[0];
240
- if (value === '\n') {
241
- curLineCount++;
242
- lastLineBreakOffset = curOffset + 1;
243
- }
244
- else {
245
- if (lastTokenEndOffset !== curOffset) {
246
- let token;
247
- if (lastTokenEndLine === curLineCount) {
248
- const colCount = curOffset - lastTokenEndOffset;
249
- if (colCount < smallTextTokens0Line.length) {
250
- token = smallTextTokens0Line[colCount];
251
- }
252
- else {
253
- const length = toLength(0, colCount);
254
- token = new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
255
- }
256
- }
257
- else {
258
- const lineCount = curLineCount - lastTokenEndLine;
259
- const colCount = curOffset - lastLineBreakOffset;
260
- if (lineCount === 1 && colCount < smallTextTokens1Line.length) {
261
- token = smallTextTokens1Line[colCount];
262
- }
263
- else {
264
- const length = toLength(lineCount, colCount);
265
- token = new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
266
- }
267
- }
268
- tokens.push(token);
269
- }
270
- // value is matched by regexp, so the token must exist
271
- tokens.push(brackets.getToken(value));
272
- lastTokenEndOffset = curOffset + value.length;
273
- lastTokenEndLine = curLineCount;
274
- }
275
- }
276
- }
277
- const offset = text.length;
278
- if (lastTokenEndOffset !== offset) {
279
- const length = (lastTokenEndLine === curLineCount)
280
- ? toLength(0, offset - lastTokenEndOffset)
281
- : toLength(curLineCount - lastTokenEndLine, offset - lastLineBreakOffset);
282
- tokens.push(new Token(length, 0 /* TokenKind.Text */, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length)));
283
- }
284
- this.length = toLength(curLineCount, offset - lastLineBreakOffset);
285
- this.tokens = tokens;
286
- }
287
- get offset() {
288
- return this._offset;
289
- }
290
- read() {
291
- return this.tokens[this.idx++] || null;
292
- }
293
- peek() {
294
- return this.tokens[this.idx] || null;
295
- }
296
- skip(length) {
297
- throw new NotSupportedError();
298
- }
299
- }
300
-
301
- export { FastTokenizer, TextBufferTokenizer, Token };
@@ -1,261 +0,0 @@
1
- import { onUnexpectedError } from '../../../../base/common/errors.js';
2
- import { Emitter } from '../../../../base/common/event.js';
3
- import { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';
4
- import { countEOL } from '../../core/misc/eolCounter.js';
5
- import { Position } from '../../core/position.js';
6
- import { LineRange } from '../../core/ranges/lineRange.js';
7
- import { TokenizationRegistry } from '../../languages.js';
8
- import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';
9
- import { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';
10
- import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from '../textModelTokens.js';
11
- import { AbstractSyntaxTokenBackend, AttachedViewHandler } from './abstractSyntaxTokenBackend.js';
12
-
13
- /*---------------------------------------------------------------------------------------------
14
- * Copyright (c) Microsoft Corporation. All rights reserved.
15
- * Licensed under the MIT License. See License.txt in the project root for license information.
16
- *--------------------------------------------------------------------------------------------*/
17
- /** For TextMate */
18
- class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
19
- constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
20
- super(languageIdCodec, textModel);
21
- this.getLanguageId = getLanguageId;
22
- this._tokenizer = null;
23
- this._backgroundTokenizationState = 1 /* BackgroundTokenizationState.InProgress */;
24
- this._onDidChangeBackgroundTokenizationState = this._register(new Emitter());
25
- this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
26
- this._defaultBackgroundTokenizer = null;
27
- this._backgroundTokenizer = this._register(new MutableDisposable());
28
- this._tokens = new ContiguousTokensStore(this._languageIdCodec);
29
- this._debugBackgroundTokenizer = this._register(new MutableDisposable());
30
- this._attachedViewStates = this._register(new DisposableMap());
31
- this._register(TokenizationRegistry.onDidChange((e) => {
32
- const languageId = this.getLanguageId();
33
- if (e.changedLanguages.indexOf(languageId) === -1) {
34
- return;
35
- }
36
- this.todo_resetTokenization();
37
- }));
38
- this.todo_resetTokenization();
39
- this._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {
40
- if (state) {
41
- let existing = this._attachedViewStates.get(view);
42
- if (!existing) {
43
- existing = new AttachedViewHandler(() => this.refreshRanges(existing.lineRanges));
44
- this._attachedViewStates.set(view, existing);
45
- }
46
- existing.handleStateChange(state);
47
- }
48
- else {
49
- this._attachedViewStates.deleteAndDispose(view);
50
- }
51
- }));
52
- }
53
- todo_resetTokenization(fireTokenChangeEvent = true) {
54
- this._tokens.flush();
55
- this._debugBackgroundTokens?.flush();
56
- if (this._debugBackgroundStates) {
57
- this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
58
- }
59
- if (fireTokenChangeEvent) {
60
- this._onDidChangeTokens.fire({
61
- semanticTokensApplied: false,
62
- ranges: [
63
- {
64
- fromLineNumber: 1,
65
- toLineNumber: this._textModel.getLineCount(),
66
- },
67
- ],
68
- });
69
- }
70
- const initializeTokenization = () => {
71
- if (this._textModel.isTooLargeForTokenization()) {
72
- return [null, null];
73
- }
74
- const tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());
75
- if (!tokenizationSupport) {
76
- return [null, null];
77
- }
78
- let initialState;
79
- try {
80
- initialState = tokenizationSupport.getInitialState();
81
- }
82
- catch (e) {
83
- onUnexpectedError(e);
84
- return [null, null];
85
- }
86
- return [tokenizationSupport, initialState];
87
- };
88
- const [tokenizationSupport, initialState] = initializeTokenization();
89
- if (tokenizationSupport && initialState) {
90
- this._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);
91
- }
92
- else {
93
- this._tokenizer = null;
94
- }
95
- this._backgroundTokenizer.clear();
96
- this._defaultBackgroundTokenizer = null;
97
- if (this._tokenizer) {
98
- const b = {
99
- setTokens: (tokens) => {
100
- this.setTokens(tokens);
101
- },
102
- backgroundTokenizationFinished: () => {
103
- if (this._backgroundTokenizationState === 2 /* BackgroundTokenizationState.Completed */) {
104
- // We already did a full tokenization and don't go back to progressing.
105
- return;
106
- }
107
- const newState = 2 /* BackgroundTokenizationState.Completed */;
108
- this._backgroundTokenizationState = newState;
109
- this._onDidChangeBackgroundTokenizationState.fire();
110
- },
111
- setEndState: (lineNumber, state) => {
112
- if (!this._tokenizer) {
113
- return;
114
- }
115
- const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
116
- // Don't accept states for definitely valid states, the renderer is ahead of the worker!
117
- if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
118
- this._tokenizer?.store.setEndState(lineNumber, state);
119
- }
120
- },
121
- };
122
- if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
123
- this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
124
- }
125
- if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {
126
- this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
127
- new DefaultBackgroundTokenizer(this._tokenizer, b);
128
- this._defaultBackgroundTokenizer.handleChanges();
129
- }
130
- if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
131
- this._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);
132
- this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
133
- this._debugBackgroundTokenizer.clear();
134
- this._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {
135
- setTokens: (tokens) => {
136
- this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
137
- },
138
- backgroundTokenizationFinished() {
139
- // NO OP
140
- },
141
- setEndState: (lineNumber, state) => {
142
- this._debugBackgroundStates?.setEndState(lineNumber, state);
143
- },
144
- });
145
- }
146
- else {
147
- this._debugBackgroundTokens = undefined;
148
- this._debugBackgroundStates = undefined;
149
- this._debugBackgroundTokenizer.value = undefined;
150
- }
151
- }
152
- this.refreshAllVisibleLineTokens();
153
- }
154
- handleDidChangeAttached() {
155
- this._defaultBackgroundTokenizer?.handleChanges();
156
- }
157
- handleDidChangeContent(e) {
158
- if (e.isFlush) {
159
- // Don't fire the event, as the view might not have got the text change event yet
160
- this.todo_resetTokenization(false);
161
- }
162
- else if (!e.isEolChange) { // We don't have to do anything on an EOL change
163
- for (const c of e.changes) {
164
- const [eolCount, firstLineLength] = countEOL(c.text);
165
- this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
166
- this._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);
167
- }
168
- this._debugBackgroundStates?.acceptChanges(e.changes);
169
- if (this._tokenizer) {
170
- this._tokenizer.store.acceptChanges(e.changes);
171
- }
172
- this._defaultBackgroundTokenizer?.handleChanges();
173
- }
174
- }
175
- setTokens(tokens) {
176
- const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
177
- if (changes.length > 0) {
178
- this._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });
179
- }
180
- return { changes: changes };
181
- }
182
- refreshAllVisibleLineTokens() {
183
- const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));
184
- this.refreshRanges(ranges);
185
- }
186
- refreshRanges(ranges) {
187
- for (const range of ranges) {
188
- this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
189
- }
190
- }
191
- refreshRange(startLineNumber, endLineNumber) {
192
- if (!this._tokenizer) {
193
- return;
194
- }
195
- startLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));
196
- endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
197
- const builder = new ContiguousMultilineTokensBuilder();
198
- const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
199
- const changedTokens = this.setTokens(builder.finalize());
200
- if (heuristicTokens) {
201
- // We overrode tokens with heuristically computed ones.
202
- // Because old states might get reused (thus stopping invalidation),
203
- // we have to explicitly request the tokens for the changed ranges again.
204
- for (const c of changedTokens.changes) {
205
- this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
206
- }
207
- }
208
- this._defaultBackgroundTokenizer?.checkFinished();
209
- }
210
- forceTokenization(lineNumber) {
211
- const builder = new ContiguousMultilineTokensBuilder();
212
- this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
213
- this.setTokens(builder.finalize());
214
- this._defaultBackgroundTokenizer?.checkFinished();
215
- }
216
- hasAccurateTokensForLine(lineNumber) {
217
- if (!this._tokenizer) {
218
- return true;
219
- }
220
- return this._tokenizer.hasAccurateTokensForLine(lineNumber);
221
- }
222
- isCheapToTokenize(lineNumber) {
223
- if (!this._tokenizer) {
224
- return true;
225
- }
226
- return this._tokenizer.isCheapToTokenize(lineNumber);
227
- }
228
- getLineTokens(lineNumber) {
229
- const lineText = this._textModel.getLineContent(lineNumber);
230
- const result = this._tokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
231
- if (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {
232
- if (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {
233
- const backgroundResult = this._debugBackgroundTokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
234
- if (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {
235
- this._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);
236
- }
237
- }
238
- }
239
- return result;
240
- }
241
- getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
242
- if (!this._tokenizer) {
243
- return 0 /* StandardTokenType.Other */;
244
- }
245
- const position = this._textModel.validatePosition(new Position(lineNumber, column));
246
- this.forceTokenization(position.lineNumber);
247
- return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
248
- }
249
- tokenizeLinesAt(lineNumber, lines) {
250
- if (!this._tokenizer) {
251
- return null;
252
- }
253
- this.forceTokenization(lineNumber);
254
- return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
255
- }
256
- get hasTokens() {
257
- return this._tokens.hasTokens;
258
- }
259
- }
260
-
261
- export { TokenizerSyntaxTokenBackend };
@@ -1,9 +0,0 @@
1
- import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
2
-
3
- /*---------------------------------------------------------------------------------------------
4
- * Copyright (c) Microsoft Corporation. All rights reserved.
5
- * Licensed under the MIT License. See License.txt in the project root for license information.
6
- *--------------------------------------------------------------------------------------------*/
7
- const ITreeSitterLibraryService = createDecorator('treeSitterLibraryService');
8
-
9
- export { ITreeSitterLibraryService };