monaco-editor-core 0.56.0-dev-20251124 → 0.56.0-dev-20251217

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/esm/nls.keys.json +1 -1
  2. package/esm/nls.messages.cs.js +2 -2
  3. package/esm/nls.messages.de.js +2 -2
  4. package/esm/nls.messages.es.js +2 -2
  5. package/esm/nls.messages.fr.js +2 -2
  6. package/esm/nls.messages.it.js +2 -2
  7. package/esm/nls.messages.ja.js +2 -2
  8. package/esm/nls.messages.js +1 -1
  9. package/esm/nls.messages.json +1 -1
  10. package/esm/nls.messages.ko.js +2 -2
  11. package/esm/nls.messages.pl.js +2 -2
  12. package/esm/nls.messages.pt-br.js +2 -2
  13. package/esm/nls.messages.ru.js +2 -2
  14. package/esm/nls.messages.tr.js +2 -2
  15. package/esm/nls.messages.zh-cn.js +2 -2
  16. package/esm/nls.messages.zh-tw.js +2 -2
  17. package/esm/nls.metadata.json +34 -2
  18. package/esm/vs/base/browser/dom.js +1 -4
  19. package/esm/vs/base/browser/dom.js.map +1 -1
  20. package/esm/vs/base/browser/markdownRenderer.js +25 -11
  21. package/esm/vs/base/browser/markdownRenderer.js.map +1 -1
  22. package/esm/vs/base/browser/ui/button/button.js +15 -8
  23. package/esm/vs/base/browser/ui/button/button.js.map +1 -1
  24. package/esm/vs/base/browser/ui/codicons/codicon/codicon-modifiers.css +3 -8
  25. package/esm/vs/base/browser/ui/codicons/codicon/codicon.ttf +0 -0
  26. package/esm/vs/base/browser/ui/list/list.css +1 -0
  27. package/esm/vs/base/browser/ui/toolbar/toolbar.js +3 -0
  28. package/esm/vs/base/browser/ui/toolbar/toolbar.js.map +1 -1
  29. package/esm/vs/base/common/async.js +11 -0
  30. package/esm/vs/base/common/async.js.map +1 -1
  31. package/esm/vs/base/common/codiconsLibrary.js +11 -2
  32. package/esm/vs/base/common/codiconsLibrary.js.map +1 -1
  33. package/esm/vs/base/common/defaultAccount.js +6 -0
  34. package/esm/vs/base/common/defaultAccount.js.map +1 -0
  35. package/esm/vs/base/common/equals.js +73 -24
  36. package/esm/vs/base/common/equals.js.map +1 -1
  37. package/esm/vs/base/common/event.js +14 -4
  38. package/esm/vs/base/common/event.js.map +1 -1
  39. package/esm/vs/base/common/iterator.js.map +1 -1
  40. package/esm/vs/base/common/keybindings.js +0 -8
  41. package/esm/vs/base/common/keybindings.js.map +1 -1
  42. package/esm/vs/base/common/lifecycle.js +2 -2
  43. package/esm/vs/base/common/lifecycle.js.map +1 -1
  44. package/esm/vs/base/common/marshallingIds.js.map +1 -1
  45. package/esm/vs/base/common/observableInternal/index.js +1 -1
  46. package/esm/vs/base/common/observableInternal/index.js.map +1 -1
  47. package/esm/vs/base/common/observableInternal/observables/observableFromEvent.js +1 -1
  48. package/esm/vs/base/common/observableInternal/observables/observableFromEvent.js.map +1 -1
  49. package/esm/vs/base/common/observableInternal/reactions/autorunImpl.js +16 -0
  50. package/esm/vs/base/common/observableInternal/reactions/autorunImpl.js.map +1 -1
  51. package/esm/vs/base/common/observableInternal/utils/utils.js +11 -0
  52. package/esm/vs/base/common/observableInternal/utils/utils.js.map +1 -1
  53. package/esm/vs/base/common/product.js.map +1 -1
  54. package/esm/vs/base/common/strings.js +3 -1
  55. package/esm/vs/base/common/strings.js.map +1 -1
  56. package/esm/vs/editor/browser/controller/dragScrolling.js +2 -1
  57. package/esm/vs/editor/browser/controller/dragScrolling.js.map +1 -1
  58. package/esm/vs/editor/browser/controller/editContext/clipboardUtils.js +73 -2
  59. package/esm/vs/editor/browser/controller/editContext/clipboardUtils.js.map +1 -1
  60. package/esm/vs/editor/browser/controller/editContext/native/nativeEditContext.js +7 -51
  61. package/esm/vs/editor/browser/controller/editContext/native/nativeEditContext.js.map +1 -1
  62. package/esm/vs/editor/browser/controller/editContext/screenReaderUtils.js.map +1 -1
  63. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContext.js +7 -20
  64. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContext.js.map +1 -1
  65. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextInput.js +14 -44
  66. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextInput.js.map +1 -1
  67. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextRegistry.js +19 -0
  68. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextRegistry.js.map +1 -0
  69. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextState.js +14 -1
  70. package/esm/vs/editor/browser/controller/editContext/textArea/textAreaEditContextState.js.map +1 -1
  71. package/esm/vs/editor/browser/controller/mouseHandler.js +5 -2
  72. package/esm/vs/editor/browser/controller/mouseHandler.js.map +1 -1
  73. package/esm/vs/editor/browser/editorBrowser.js.map +1 -1
  74. package/esm/vs/editor/browser/observableCodeEditor.js +32 -15
  75. package/esm/vs/editor/browser/observableCodeEditor.js.map +1 -1
  76. package/esm/vs/editor/browser/view.js +11 -2
  77. package/esm/vs/editor/browser/view.js.map +1 -1
  78. package/esm/vs/editor/browser/viewParts/contentWidgets/contentWidgets.js +5 -2
  79. package/esm/vs/editor/browser/viewParts/contentWidgets/contentWidgets.js.map +1 -1
  80. package/esm/vs/editor/browser/widget/codeEditor/codeEditorWidget.js +10 -0
  81. package/esm/vs/editor/browser/widget/codeEditor/codeEditorWidget.js.map +1 -1
  82. package/esm/vs/editor/browser/widget/multiDiffEditor/multiDiffEditorWidgetImpl.js.map +1 -1
  83. package/esm/vs/editor/browser/widget/multiDiffEditor/style.css +48 -0
  84. package/esm/vs/editor/common/config/fontInfo.js +1 -1
  85. package/esm/vs/editor/common/config/fontInfo.js.map +1 -1
  86. package/esm/vs/editor/common/core/edits/edit.js +19 -0
  87. package/esm/vs/editor/common/core/edits/edit.js.map +1 -1
  88. package/esm/vs/editor/common/core/edits/stringEdit.js +13 -0
  89. package/esm/vs/editor/common/core/edits/stringEdit.js.map +1 -1
  90. package/esm/vs/editor/common/core/edits/textEdit.js.map +1 -1
  91. package/esm/vs/editor/common/core/text/positionToOffsetImpl.js +24 -10
  92. package/esm/vs/editor/common/core/text/positionToOffsetImpl.js.map +1 -1
  93. package/esm/vs/editor/common/cursor/cursorMoveCommands.js +4 -4
  94. package/esm/vs/editor/common/cursor/cursorMoveCommands.js.map +1 -1
  95. package/esm/vs/editor/common/cursor/cursorWordOperations.js +6 -3
  96. package/esm/vs/editor/common/cursor/cursorWordOperations.js.map +1 -1
  97. package/esm/vs/editor/common/languages/defaultDocumentColorsComputer.js +2 -2
  98. package/esm/vs/editor/common/languages/defaultDocumentColorsComputer.js.map +1 -1
  99. package/esm/vs/editor/common/languages/nullTokenize.js +1 -1
  100. package/esm/vs/editor/common/languages/nullTokenize.js.map +1 -1
  101. package/esm/vs/editor/common/languages/supports/tokenization.js +3 -0
  102. package/esm/vs/editor/common/languages/supports/tokenization.js.map +1 -1
  103. package/esm/vs/editor/common/languages.js +2 -1
  104. package/esm/vs/editor/common/languages.js.map +1 -1
  105. package/esm/vs/editor/common/model/decorationProvider.js +21 -1
  106. package/esm/vs/editor/common/model/decorationProvider.js.map +1 -1
  107. package/esm/vs/editor/common/model/textModel.js +49 -30
  108. package/esm/vs/editor/common/model/textModel.js.map +1 -1
  109. package/esm/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.js +6 -3
  110. package/esm/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.js.map +1 -1
  111. package/esm/vs/editor/common/model/tokens/annotations.js +177 -0
  112. package/esm/vs/editor/common/model/tokens/annotations.js.map +1 -0
  113. package/esm/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.js +131 -0
  114. package/esm/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.js.map +1 -0
  115. package/esm/vs/editor/common/model/tokens/tokenizationTextModelPart.js +9 -0
  116. package/esm/vs/editor/common/model/tokens/tokenizationTextModelPart.js.map +1 -1
  117. package/esm/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.js +9 -0
  118. package/esm/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.js.map +1 -1
  119. package/esm/vs/editor/common/textModelEditSource.js +2 -0
  120. package/esm/vs/editor/common/textModelEditSource.js.map +1 -1
  121. package/esm/vs/editor/common/textModelEvents.js.map +1 -1
  122. package/esm/vs/editor/common/viewLayout/lineDecorations.js.map +1 -1
  123. package/esm/vs/editor/common/viewLayout/viewLineRenderer.js +1 -1
  124. package/esm/vs/editor/common/viewLayout/viewLineRenderer.js.map +1 -1
  125. package/esm/vs/editor/contrib/clipboard/browser/clipboard.js +54 -36
  126. package/esm/vs/editor/contrib/clipboard/browser/clipboard.js.map +1 -1
  127. package/esm/vs/editor/contrib/dropOrPasteInto/browser/copyPasteController.js +14 -1
  128. package/esm/vs/editor/contrib/dropOrPasteInto/browser/copyPasteController.js.map +1 -1
  129. package/esm/vs/editor/contrib/floatingMenu/browser/floatingMenu.css +1 -1
  130. package/esm/vs/editor/contrib/hover/browser/hover.css +1 -1
  131. package/esm/vs/editor/contrib/hover/browser/markdownHoverParticipant.js +1 -1
  132. package/esm/vs/editor/contrib/hover/browser/markdownHoverParticipant.js.map +1 -1
  133. package/esm/vs/editor/contrib/inPlaceReplace/browser/inPlaceReplace.js +2 -2
  134. package/esm/vs/editor/contrib/inlayHints/browser/inlayHintsController.js +1 -0
  135. package/esm/vs/editor/contrib/inlayHints/browser/inlayHintsController.js.map +1 -1
  136. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/commandIds.js +2 -0
  137. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/commandIds.js.map +1 -1
  138. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/commands.js +33 -4
  139. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/commands.js.map +1 -1
  140. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionContextKeys.js +13 -12
  141. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionContextKeys.js.map +1 -1
  142. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.js +7 -1
  143. package/esm/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.js.map +1 -1
  144. package/esm/vs/editor/contrib/inlineCompletions/browser/hintsWidget/hoverParticipant.js +2 -2
  145. package/esm/vs/editor/contrib/inlineCompletions/browser/hintsWidget/inlineCompletionsHintsWidget.js +5 -5
  146. package/esm/vs/editor/contrib/inlineCompletions/browser/inlineCompletions.contribution.js +2 -1
  147. package/esm/vs/editor/contrib/inlineCompletions/browser/inlineCompletions.contribution.js.map +1 -1
  148. package/esm/vs/editor/contrib/inlineCompletions/browser/model/InlineSuggestAlternativeAction.js +8 -0
  149. package/esm/vs/editor/contrib/inlineCompletions/browser/model/InlineSuggestAlternativeAction.js.map +1 -0
  150. package/esm/vs/editor/contrib/inlineCompletions/browser/model/editKind.js +198 -0
  151. package/esm/vs/editor/contrib/inlineCompletions/browser/model/editKind.js.map +1 -0
  152. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionIsVisible.js +40 -0
  153. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionIsVisible.js.map +1 -0
  154. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.js +128 -65
  155. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.js.map +1 -1
  156. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsSource.js +69 -13
  157. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsSource.js.map +1 -1
  158. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineSuggestionItem.js +159 -94
  159. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineSuggestionItem.js.map +1 -1
  160. package/esm/vs/editor/contrib/inlineCompletions/browser/model/provideInlineCompletions.js +124 -50
  161. package/esm/vs/editor/contrib/inlineCompletions/browser/model/provideInlineCompletions.js.map +1 -1
  162. package/esm/vs/editor/contrib/inlineCompletions/browser/model/renameSymbolProcessor.js +393 -0
  163. package/esm/vs/editor/contrib/inlineCompletions/browser/model/renameSymbolProcessor.js.map +1 -0
  164. package/esm/vs/editor/contrib/inlineCompletions/browser/telemetry.js.map +1 -1
  165. package/esm/vs/editor/contrib/inlineCompletions/browser/utils.js +20 -0
  166. package/esm/vs/editor/contrib/inlineCompletions/browser/utils.js.map +1 -1
  167. package/esm/vs/editor/contrib/inlineCompletions/browser/view/ghostText/ghostTextView.js +59 -42
  168. package/esm/vs/editor/contrib/inlineCompletions/browser/view/ghostText/ghostTextView.js.map +1 -1
  169. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorMenu.js +29 -8
  170. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorMenu.js.map +1 -1
  171. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.js +78 -46
  172. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.js.map +1 -1
  173. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditWithChanges.js +9 -4
  174. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditWithChanges.js.map +1 -1
  175. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsModel.js +11 -4
  176. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsModel.js.map +1 -1
  177. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsView.js +81 -41
  178. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsView.js.map +1 -1
  179. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewInterface.js +48 -0
  180. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewInterface.js.map +1 -1
  181. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.js +18 -15
  182. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.js.map +1 -1
  183. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsCollapsedView.js +3 -4
  184. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsCollapsedView.js.map +1 -1
  185. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsCustomView.js +6 -7
  186. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsCustomView.js.map +1 -1
  187. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsDeletionView.js +4 -5
  188. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsDeletionView.js.map +1 -1
  189. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsInsertionView.js +4 -3
  190. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsInsertionView.js.map +1 -1
  191. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsLineReplacementView.js +8 -8
  192. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsLineReplacementView.js.map +1 -1
  193. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsSideBySideView.js +9 -7
  194. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsSideBySideView.js.map +1 -1
  195. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsWordReplacementView.js +203 -41
  196. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/inlineEditsWordReplacementView.js.map +1 -1
  197. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/jumpToView.js +189 -0
  198. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/jumpToView.js.map +1 -0
  199. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/inlineEditsLongDistanceHint.js +85 -102
  200. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/inlineEditsLongDistanceHint.js.map +1 -1
  201. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/longDistancePreviewEditor.js +75 -32
  202. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/longDistancePreviewEditor.js.map +1 -1
  203. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/longDistnaceWidgetPlacement.js +121 -0
  204. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/longDistanceHint/longDistnaceWidgetPlacement.js.map +1 -0
  205. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/originalEditorInlineDiffView.js +2 -1
  206. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViews/originalEditorInlineDiffView.js.map +1 -1
  207. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/theme.js +22 -20
  208. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/theme.js.map +1 -1
  209. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/utils/utils.js +15 -5
  210. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/utils/utils.js.map +1 -1
  211. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/view.css +39 -6
  212. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineSuggestionsView.js +10 -5
  213. package/esm/vs/editor/contrib/inlineCompletions/browser/view/inlineSuggestionsView.js.map +1 -1
  214. package/esm/vs/editor/contrib/insertFinalNewLine/browser/insertFinalNewLine.js +1 -1
  215. package/esm/vs/editor/contrib/lineSelection/browser/lineSelection.js +1 -1
  216. package/esm/vs/editor/contrib/linesOperations/browser/linesOperations.js +32 -32
  217. package/esm/vs/editor/contrib/linesOperations/browser/linesOperations.js.map +1 -1
  218. package/esm/vs/editor/contrib/linkedEditing/browser/linkedEditing.js +2 -2
  219. package/esm/vs/editor/contrib/links/browser/links.js +10 -10
  220. package/esm/vs/editor/contrib/message/browser/messageController.js +1 -1
  221. package/esm/vs/editor/contrib/multicursor/browser/multicursor.js +22 -22
  222. package/esm/vs/editor/contrib/parameterHints/browser/parameterHints.js +1 -1
  223. package/esm/vs/editor/contrib/parameterHints/browser/parameterHintsWidget.js +4 -4
  224. package/esm/vs/editor/contrib/peekView/browser/peekView.js +18 -18
  225. package/esm/vs/editor/contrib/placeholderText/browser/placeholderText.contribution.js +1 -1
  226. package/esm/vs/editor/contrib/quickAccess/browser/gotoLineQuickAccess.js +11 -11
  227. package/esm/vs/editor/contrib/quickAccess/browser/gotoSymbolQuickAccess.js +32 -32
  228. package/esm/vs/editor/contrib/readOnlyMessage/browser/contribution.js +2 -2
  229. package/esm/vs/editor/contrib/rename/browser/rename.js +19 -11
  230. package/esm/vs/editor/contrib/rename/browser/rename.js.map +1 -1
  231. package/esm/vs/editor/contrib/rename/browser/renameWidget.js +7 -7
  232. package/esm/vs/editor/contrib/sectionHeaders/browser/sectionHeaders.js +3 -2
  233. package/esm/vs/editor/contrib/sectionHeaders/browser/sectionHeaders.js.map +1 -1
  234. package/esm/vs/editor/contrib/smartSelect/browser/smartSelect.js +4 -4
  235. package/esm/vs/editor/contrib/snippet/browser/snippetController2.js +4 -4
  236. package/esm/vs/editor/contrib/snippet/browser/snippetParser.js +33 -0
  237. package/esm/vs/editor/contrib/snippet/browser/snippetParser.js.map +1 -1
  238. package/esm/vs/editor/contrib/snippet/browser/snippetVariables.js +4 -4
  239. package/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollActions.js +11 -11
  240. package/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollController.js +12 -4
  241. package/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollController.js.map +1 -1
  242. package/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollWidget.js +1 -0
  243. package/esm/vs/editor/contrib/stickyScroll/browser/stickyScrollWidget.js.map +1 -1
  244. package/esm/vs/editor/contrib/suggest/browser/suggest.js +9 -8
  245. package/esm/vs/editor/contrib/suggest/browser/suggest.js.map +1 -1
  246. package/esm/vs/editor/contrib/suggest/browser/suggestController.js +39 -28
  247. package/esm/vs/editor/contrib/suggest/browser/suggestController.js.map +1 -1
  248. package/esm/vs/editor/contrib/suggest/browser/suggestInlineCompletions.js +1 -0
  249. package/esm/vs/editor/contrib/suggest/browser/suggestInlineCompletions.js.map +1 -1
  250. package/esm/vs/editor/contrib/suggest/browser/suggestWidget.js +22 -18
  251. package/esm/vs/editor/contrib/suggest/browser/suggestWidget.js.map +1 -1
  252. package/esm/vs/editor/contrib/suggest/browser/suggestWidgetDetails.js +28 -6
  253. package/esm/vs/editor/contrib/suggest/browser/suggestWidgetDetails.js.map +1 -1
  254. package/esm/vs/editor/contrib/suggest/browser/suggestWidgetRenderer.js +2 -2
  255. package/esm/vs/editor/contrib/suggest/browser/wordContextKey.js +1 -1
  256. package/esm/vs/editor/contrib/symbolIcons/browser/symbolIcons.js +33 -33
  257. package/esm/vs/editor/contrib/toggleTabFocusMode/browser/toggleTabFocusMode.js +4 -4
  258. package/esm/vs/editor/contrib/tokenization/browser/tokenization.js +1 -1
  259. package/esm/vs/editor/contrib/unicodeHighlighter/browser/bannerController.js +1 -1
  260. package/esm/vs/editor/contrib/unicodeHighlighter/browser/unicodeHighlighter.js +24 -24
  261. package/esm/vs/editor/contrib/unusualLineTerminators/browser/unusualLineTerminators.js +5 -5
  262. package/esm/vs/editor/contrib/wordHighlighter/browser/highlightDecorations.js +9 -9
  263. package/esm/vs/editor/contrib/wordHighlighter/browser/wordHighlighter.js +3 -3
  264. package/esm/vs/editor/contrib/wordOperations/browser/wordOperations.js +19 -2
  265. package/esm/vs/editor/contrib/wordOperations/browser/wordOperations.js.map +1 -1
  266. package/esm/vs/editor/editor.api.d.ts +37 -2
  267. package/esm/vs/editor/standalone/browser/standaloneLanguages.js +2 -2
  268. package/esm/vs/editor/standalone/browser/standaloneLanguages.js.map +1 -1
  269. package/esm/vs/editor/standalone/browser/standaloneServices.js +11 -1
  270. package/esm/vs/editor/standalone/browser/standaloneServices.js.map +1 -1
  271. package/esm/vs/editor/standalone/common/monarch/monarchLexer.js +1 -1
  272. package/esm/vs/editor/standalone/common/monarch/monarchLexer.js.map +1 -1
  273. package/esm/vs/platform/accessibilitySignal/browser/accessibilitySignalService.js +62 -62
  274. package/esm/vs/platform/action/common/actionCommonCategories.js +6 -6
  275. package/esm/vs/platform/actionWidget/browser/actionList.js +4 -4
  276. package/esm/vs/platform/actionWidget/browser/actionWidget.css +11 -8
  277. package/esm/vs/platform/actionWidget/browser/actionWidget.js +7 -7
  278. package/esm/vs/platform/actions/browser/menuEntryActionViewItem.js +5 -5
  279. package/esm/vs/platform/actions/browser/toolbar.js +2 -2
  280. package/esm/vs/platform/actions/common/actions.js +11 -4
  281. package/esm/vs/platform/actions/common/actions.js.map +1 -1
  282. package/esm/vs/platform/actions/common/menuService.js +2 -2
  283. package/esm/vs/platform/configuration/common/configurationRegistry.js +10 -10
  284. package/esm/vs/platform/contextkey/browser/contextKeyService.js +1 -1
  285. package/esm/vs/platform/contextkey/common/contextkey.js +9 -9
  286. package/esm/vs/platform/contextkey/common/contextkeys.js +9 -9
  287. package/esm/vs/platform/contextkey/common/scanner.js +5 -5
  288. package/esm/vs/platform/{product/common/productService.js → defaultAccount/common/defaultAccount.js} +2 -2
  289. package/esm/vs/platform/defaultAccount/common/defaultAccount.js.map +1 -0
  290. package/esm/vs/platform/history/browser/contextScopedHistoryWidget.js +1 -1
  291. package/esm/vs/platform/hover/browser/hoverWidget.js +1 -1
  292. package/esm/vs/platform/hover/browser/updatableHoverWidget.js +1 -1
  293. package/esm/vs/platform/instantiation/common/instantiation.js.map +1 -1
  294. package/esm/vs/platform/instantiation/common/instantiationService.js +1 -8
  295. package/esm/vs/platform/instantiation/common/instantiationService.js.map +1 -1
  296. package/esm/vs/platform/keybinding/common/abstractKeybindingService.js +4 -4
  297. package/esm/vs/platform/list/browser/listService.js +27 -27
  298. package/esm/vs/platform/log/common/log.js +1 -1
  299. package/esm/vs/platform/log/common/log.js.map +1 -1
  300. package/esm/vs/platform/markers/common/markerService.js +6 -6
  301. package/esm/vs/platform/markers/common/markerService.js.map +1 -1
  302. package/esm/vs/platform/markers/common/markers.js +6 -6
  303. package/esm/vs/platform/quickinput/browser/commandsQuickAccess.js +8 -8
  304. package/esm/vs/platform/quickinput/browser/helpQuickAccess.js +1 -1
  305. package/esm/vs/platform/quickinput/browser/media/quickInput.css +7 -3
  306. package/esm/vs/platform/quickinput/browser/quickInput.js +19 -11
  307. package/esm/vs/platform/quickinput/browser/quickInput.js.map +1 -1
  308. package/esm/vs/platform/quickinput/browser/quickInputActions.js +5 -5
  309. package/esm/vs/platform/quickinput/browser/quickInputController.js +6 -6
  310. package/esm/vs/platform/quickinput/browser/quickInputList.js +1 -1
  311. package/esm/vs/platform/quickinput/browser/quickInputUtils.js +1 -1
  312. package/esm/vs/platform/quickinput/browser/tree/quickInputTreeAccessibilityProvider.js +1 -1
  313. package/esm/vs/platform/theme/common/colorUtils.js +2 -2
  314. package/esm/vs/platform/theme/common/colors/baseColors.js +18 -17
  315. package/esm/vs/platform/theme/common/colors/baseColors.js.map +1 -1
  316. package/esm/vs/platform/theme/common/colors/chartsColors.js +8 -8
  317. package/esm/vs/platform/theme/common/colors/editorColors.js +95 -95
  318. package/esm/vs/platform/theme/common/colors/inputColors.js +47 -47
  319. package/esm/vs/platform/theme/common/colors/listColors.js +36 -36
  320. package/esm/vs/platform/theme/common/colors/menuColors.js +7 -7
  321. package/esm/vs/platform/theme/common/colors/minimapColors.js +13 -13
  322. package/esm/vs/platform/theme/common/colors/minimapColors.js.map +1 -1
  323. package/esm/vs/platform/theme/common/colors/miscColors.js +16 -16
  324. package/esm/vs/platform/theme/common/colors/quickpickColors.js +9 -9
  325. package/esm/vs/platform/theme/common/colors/searchColors.js +3 -3
  326. package/esm/vs/platform/theme/common/iconRegistry.js +6 -6
  327. package/esm/vs/platform/undoRedo/common/undoRedoService.js +21 -21
  328. package/esm/vs/platform/undoRedo/common/undoRedoService.js.map +1 -1
  329. package/esm/vs/platform/window/common/window.js +1 -1
  330. package/esm/vs/platform/window/common/window.js.map +1 -1
  331. package/esm/vs/platform/workspace/common/workspace.js +1 -1
  332. package/monaco.d.ts +37 -2
  333. package/package.json +2 -2
  334. package/version.txt +1 -1
  335. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineEdit.js +0 -16
  336. package/esm/vs/editor/contrib/inlineCompletions/browser/model/inlineEdit.js.map +0 -1
  337. package/esm/vs/platform/product/common/productService.js.map +0 -1
package/esm/vs/editor/common/model/tokens/tokenizationTextModelPart.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts","vs/editor/common/model/tokens/tokenizationTextModelPart.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;;;;;;;;;;;AAGhG,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,OAAO,EAAS,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAa,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAmB,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,6BAA6B,EAA0E,MAAM,kDAAkD,CAAC;AAGzK,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,4BAA4B,EAAE,MAAM,8CAA8C,CAAC;AAK5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,qBAAqB,EAAE,MAAM,4DAA4D,CAAC;AACnG,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,uDAAuD,CAAC;AAClG,OAAO,EAAE,OAAO,EAAoC,eAAe,EAAE,MAAM,uCAAuC,CAAC;AAE5G,IAAM,yBAAyB,iCAA/B,MAAM,yBAA0B,SAAQ,aAAa;IAgB3D,YACkB,UAAqB,EACrB,0BAAqD,EAC9D,WAAmB,EACV,cAA6B,EACX,gBAAkC,EACrB,6BAA4D,EACpE,qBAA4C,EACxC,yBAAoD;QAEhG,KAAK,EAAE,CAAC;QATS,eAAU,GAAV,UAAU,CAAW;QACrB,+BAA0B,GAA1B,0BAA0B,CAA2B;QAC9D,gBAAW,GAAX,WAAW,CAAQ;QACV,mBAAc,GAAd,cAAc,CAAe;QACX,qBAAgB,GAAhB,gBAAgB,CAAkB;QACrB,kCAA6B,GAA7B,6BAA6B,CAA+B;QACpE,0BAAqB,GAArB,qBAAqB,CAAuB;QACxC,8BAAyB,GAAzB,yBAAyB,CAA2B;QAIhG,IAAI,CAAC,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9D,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YACpC,IAAI,MAAkC,CAAC;YACvC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAClE,4BAA4B,EAC5B,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,gBAAgB,CAAC,eAAe,EACrC,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACrC,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,2BAA2B,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACjK,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;gBAC7C,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC,CAAC;YAEJ,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,sCAAsC,CAAC,CAAC,CAAC,EAAE;gBAClE,IAAI,CAAC,0BAA0B,CAAC,0CAA0C,EAAE,CAAC;YAC9E,CAAC,CAAC,CAAC,CAAC;YACJ,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,6BAA6B,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YAC9D,IAAI,SAAS,EAAE,CAAC;gBACf,wJAAwJ;gBACxJ,6CAA6C;gBAC7C,KAAK,CAAC,sBAAsB,EAAE,CAAC;YAChC,CAAC;YACD,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACpF,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA8B,CAAC,CAAC;QACtF,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,iCAAiC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA2C,CAAC,CAAC;QAChH,IAAI,CAAC,gCAAgC,GAAG,IAAI,CAAC,iCAAiC,CAAC,KAAK,CAAC;QACrF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC;IACxD,CAAC;IAEM,wCAAwC,CAAC,CAA0C;QACzF,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC;IACF,CAAC;IAEM,sBAAsB,CAAC,CAA4B;QACzD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;aAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,gDAAgD;YAC5E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAErE,IAAI,CAAC,eAAe,CAAC,UAAU,CAC9B,CAAC,CAAC,KAAK,EACP,QAAQ,EACR,eAAe,EACf,cAAc,EACd,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAc,CACxD,CAAC;YACH,CAAC;QACF,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAEM,uBAAuB;QAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,uBAAuB,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACI,aAAa,CAAC,UAAkB;QACtC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IAC1E,CAAC;IAEO,4BAA4B,CAAC,CAA2B;QAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,yBAAyB;IAEjB,kBAAkB,CAAC,UAAkB;QAC5C,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;QAC9D,CAAC;IACF,CAAC;IAED,IAAW,SAAS;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;IACpC,CAAC;IAEM,iBAAiB;QACvB,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAW,2BAA2B;QACrC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,2BAA2B,CAAC;IACtD,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAEM,wBAAwB,CAAC,UAAkB;QACjD,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAC/D,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACxD,CAAC;IAEM,eAAe,CAAC,UAAkB;QACxC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAEM,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,gCAAgC,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC1F,CAAC;IAEM,eAAe,CAAC,UAAkB,EAAE,KAAe;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;IAED,aAAa;IAEb,0BAA0B;IAEnB,iBAAiB,CAAC,MAAsC,EAAE,UAAmB;QACnF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAE9D,IAAI,CAAC,4BAA4B,CAAC;YACjC,qBAAqB,EAAE,MAAM,KAAK,IAAI;YACtC,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;SAC7E,CAAC,CAAC;IACJ,CAAC;IAEM,yBAAyB;QAC/B,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;IAC1C,CAAC;IAEM,qBAAqB;QAC3B,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;IACxC,CAAC;IAEM,wBAAwB,CAAC,KAAY,EAAE,MAA+B;QAC5E,IAAI,IAAI,CAAC,yBAAyB,EAAE,EAAE,CAAC;YACtC,OAAO;QACR,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAC9C,CAAC;QAEF,IAAI,CAAC,4BAA4B,CAAC;YACjC,qBAAqB,EAAE,IAAI;YAC3B,MAAM,EAAE;gBACP;oBACC,cAAc,EAAE,YAAY,CAAC,eAAe;oBAC5C,YAAY,EAAE,YAAY,CAAC,aAAa;iBACxC;aACD;SACD,CAAC,CAAC;IACJ,CAAC;IAED,aAAa;IAEb,0BAA0B;IAEnB,iBAAiB,CAAC,SAAoB;QAC5C,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,UAAU,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,2BAAyB,CAAC,uBAAuB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/G,MAAM,eAAe,GAAG,aAAa,CACpC,QAAQ,CAAC,MAAM,EACf,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,iBAAiB,EAAE,EACvF,WAAW,CAAC,SAAS,CAAC,aAAa,EAAE,WAAW,CAAC,EACjD,aAAa,CACb,CAAC;QACF,+DAA+D;QAC/D,IACC,eAAe;YACf,eAAe,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM;YAC/C,SAAS,CAAC,MAAM,IAAI,eAAe,CAAC,SAAS,EAC5C,CAAC;YACF,OAAO,eAAe,CAAC;QACxB,CAAC;QAED,2EAA2E;QAC3E,IAAI,UAAU,GAAG,CAAC,IAAI,aAAa,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7D,2FAA2F;YAC3F,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,2BAAyB,CAAC,uBAAuB,CACrF,UAAU,EACV,UAAU,GAAG,CAAC,CACd,CAAC;YACF,MAAM,cAAc,GAAG,aAAa,CACnC,QAAQ,CAAC,MAAM,EACf,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,EAAE,EAC3F,WAAW,CAAC,SAAS,CAAC,aAAa,EAAE,WAAW,CAAC,EACjD,aAAa,CACb,CAAC;YACF,+DAA+D;YAC/D,IACC,cAAc;gBACd,cAAc,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM;gBAC9C,SAAS,CAAC,MAAM,IAAI,cAAc,CAAC,SAAS,EAC3C,CAAC;gBACF,OAAO,cAAc,CAAC;YACvB,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,wBAAwB,CAAC,UAAkB;QAClD,OAAO,IAAI,CAAC,6BAA6B,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAChF,CAAC;IAEO,MAAM,CAAC,uBAAuB,CAAC,UAAsB,EAAE,UAAkB;QAChF,MAAM,UAAU,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAExD,4CAA4C;QAC5C,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpF,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,6CAA6C;QAC7C,IAAI,SAAS,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC;QACnD,KACC,IAAI,CAAC,GAAG,UAAU,EAAE,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,EACtD,CAAC,GAAG,UAAU,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,UAAU,EAC5D,CAAC,EAAE,EACF,CAAC;YACF,SAAS,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;IAEM,oBAAoB,CAAC,QAAmB;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;QAChF,CAAC;QACD,OAAO;YACN,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC;YACjF,WAAW,EAAE,cAAc,CAAC,WAAW;YACvC,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC1B,CAAC;IACH,CAAC;IAED,aAAa;IAEb,+BAA+B;IAExB,aAAa;QACnB,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAEM,uBAAuB,CAAC,UAAkB,EAAE,MAAc;QAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3D,OAAO,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAEM,aAAa,CAAC,UAAkB,EAAE,SAAiB,KAAK;QAC9D,IAAI,IAAI,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACrC,wBAAwB;YACxB,OAAO;QACR,CAAC;QAED,MAAM,CAAC,GAA+B;YACrC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,WAAW,EAAE,UAAU;YACvB,MAAM;SACN,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,0BAA0B,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;QAE3D,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;CAGD,CAAA;AApVY,yBAAyB;IAqBnC,WAAA,gBAAgB,CAAA;IAChB,WAAA,6BAA6B,CAAA;IAC7B,WAAA,qBAAqB,CAAA;IACrB,WAAA,yBAAyB,CAAA;GAxBf,yBAAyB,CAoVrC","file":"tokenizationTextModelPart.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { CharCode } from '../../../../base/common/charCode.js';\nimport { BugIndicatingError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { IPosition, Position } from '../../core/position.js';\nimport { Range } from '../../core/range.js';\nimport { IWordAtPosition, getWordAtText } from '../../core/wordHelper.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { ILanguageService } from '../../languages/language.js';\nimport { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from '../../languages/languageConfigurationRegistry.js';\nimport { BracketPairsTextModelPart } from '../bracketPairsTextModelPart/bracketPairsImpl.js';\nimport { TextModel } from '../textModel.js';\nimport { TextModelPart } from '../textModelPart.js';\nimport { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js';\nimport { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';\nimport { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js';\nimport { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js';\nimport { SparseTokensStore } from '../../tokens/sparseTokensStore.js';\nimport { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';\nimport { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';\nimport { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';\nimport { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';\n\nexport class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {\n\tprivate readonly _semanticTokens: SparseTokensStore;\n\n\tprivate readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent>;\n\tpublic readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;\n\n\tprivate readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent>;\n\tpublic readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;\n\n\tprivate readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;\n\tpublic readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;\n\n\tpublic readonly tokens: IObservable<AbstractSyntaxTokenBackend>;\n\tprivate readonly _useTreeSitter: IObservable<boolean>;\n\tprivate readonly _languageIdObs: ISettableObservable<string>;\n\n\tconstructor(\n\t\tprivate readonly _textModel: TextModel,\n\t\tprivate readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,\n\t\tprivate _languageId: string,\n\t\tprivate readonly _attachedViews: AttachedViews,\n\t\t@ILanguageService private readonly _languageService: ILanguageService,\n\t\t@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t) {\n\t\tsuper();\n\n\t\tthis._languageIdObs = observableValue(this, this._languageId);\n\n\t\tthis._useTreeSitter = derived(this, reader => {\n\t\t\tconst languageId = this._languageIdObs.read(reader);\n\t\t\treturn this._treeSitterLibraryService.supportsLanguage(languageId, reader);\n\t\t});\n\n\t\tthis.tokens = derived(this, reader => {\n\t\t\tlet tokens: AbstractSyntaxTokenBackend;\n\t\t\tif (this._useTreeSitter.read(reader)) {\n\t\t\t\ttokens = reader.store.add(this._instantiationService.createInstance(\n\t\t\t\t\tTreeSitterSyntaxTokenBackend,\n\t\t\t\t\tthis._languageIdObs,\n\t\t\t\t\tthis._languageService.languageIdCodec,\n\t\t\t\t\tthis._textModel,\n\t\t\t\t\tthis._attachedViews.visibleLineRanges\n\t\t\t\t));\n\t\t\t} else {\n\t\t\t\ttokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));\n\t\t\t}\n\n\t\t\treader.store.add(tokens.onDidChangeTokens(e => {\n\t\t\t\tthis._emitModelTokensChangedEvent(e);\n\t\t\t}));\n\n\t\t\treader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {\n\t\t\t\tthis._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();\n\t\t\t}));\n\t\t\treturn tokens;\n\t\t});\n\n\t\tlet hadTokens = false;\n\t\tthis.tokens.recomputeInitiallyAndOnChange(this._store, value => {\n\t\t\tif (hadTokens) {\n\t\t\t\t// We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.\n\t\t\t\t// TODO@hediet: Look into why this is needed.\n\t\t\t\tvalue.todo_resetTokenization();\n\t\t\t}\n\t\t\thadTokens = true;\n\t\t});\n\n\t\tthis._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);\n\t\tthis._onDidChangeLanguage = this._register(new Emitter<IModelLanguageChangedEvent>());\n\t\tthis.onDidChangeLanguage = this._onDidChangeLanguage.event;\n\t\tthis._onDidChangeLanguageConfiguration = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>());\n\t\tthis.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;\n\t\tthis._onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());\n\t\tthis.onDidChangeTokens = this._onDidChangeTokens.event;\n\t}\n\n\tpublic handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {\n\t\tif (e.affects(this._languageId)) {\n\t\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t\t}\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\tthis._semanticTokens.flush();\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._semanticTokens.acceptEdit(\n\t\t\t\t\tc.range,\n\t\t\t\t\teolCount,\n\t\t\t\t\tfirstLineLength,\n\t\t\t\t\tlastLineLength,\n\t\t\t\t\tc.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tthis.tokens.get().handleDidChangeContent(e);\n\t}\n\n\tpublic handleDidChangeAttached(): void {\n\t\tthis.tokens.get().handleDidChangeAttached();\n\t}\n\n\t/**\n\t * Includes grammar and semantic tokens.\n\t */\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tconst syntacticTokens = this.tokens.get().getLineTokens(lineNumber);\n\t\treturn this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);\n\t}\n\n\tprivate _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {\n\t\tif (!this._textModel._isDisposing()) {\n\t\t\tthis._bracketPairsTextModelPart.handleDidChangeTokens(e);\n\t\t\tthis._onDidChangeTokens.fire(e);\n\t\t}\n\t}\n\n\t// #region Grammar Tokens\n\n\tprivate validateLineNumber(lineNumber: number): void {\n\t\tif (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {\n\t\t\tthrow new BugIndicatingError('Illegal value for lineNumber');\n\t\t}\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this.tokens.get().hasTokens;\n\t}\n\n\tpublic resetTokenization() {\n\t\tthis.tokens.get().todo_resetTokenization();\n\t}\n\n\tpublic get backgroundTokenizationState() {\n\t\treturn this.tokens.get().backgroundTokenizationState;\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().forceTokenization(lineNumber);\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic tokenizeIfCheap(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().tokenizeIfCheap(lineNumber);\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\treturn this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);\n\t}\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\treturn this.tokens.get().tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\t// #endregion\n\n\t// #region Semantic Tokens\n\n\tpublic setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {\n\t\tthis._semanticTokens.set(tokens, isComplete, this._textModel);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: tokens !== null,\n\t\t\tranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],\n\t\t});\n\t}\n\n\tpublic hasCompleteSemanticTokens(): boolean {\n\t\treturn this._semanticTokens.isComplete();\n\t}\n\n\tpublic hasSomeSemanticTokens(): boolean {\n\t\treturn !this._semanticTokens.isEmpty();\n\t}\n\n\tpublic setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {\n\t\tif (this.hasCompleteSemanticTokens()) {\n\t\t\treturn;\n\t\t}\n\t\tconst changedRange = this._textModel.validateRange(\n\t\t\tthis._semanticTokens.setPartial(range, tokens)\n\t\t);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: true,\n\t\t\tranges: [\n\t\t\t\t{\n\t\t\t\t\tfromLineNumber: changedRange.startLineNumber,\n\t\t\t\t\ttoLineNumber: changedRange.endLineNumber,\n\t\t\t\t},\n\t\t\t],\n\t\t});\n\t}\n\n\t// #endregion\n\n\t// #region Utility Methods\n\n\tpublic getWordAtPosition(_position: IPosition): IWordAtPosition | null {\n\t\tthis.assertNotDisposed();\n\n\t\tconst position = this._textModel.validatePosition(_position);\n\t\tconst lineContent = this._textModel.getLineContent(position.lineNumber);\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\tconst tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);\n\n\t\t// (1). First try checking right biased word\n\t\tconst [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);\n\t\tconst rightBiasedWord = getWordAtText(\n\t\t\tposition.column,\n\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(),\n\t\t\tlineContent.substring(rbStartOffset, rbEndOffset),\n\t\t\trbStartOffset\n\t\t);\n\t\t// Make sure the result touches the original passed in position\n\t\tif (\n\t\t\trightBiasedWord &&\n\t\t\trightBiasedWord.startColumn <= _position.column &&\n\t\t\t_position.column <= rightBiasedWord.endColumn\n\t\t) {\n\t\t\treturn rightBiasedWord;\n\t\t}\n\n\t\t// (2). Else, if we were at a language boundary, check the left biased word\n\t\tif (tokenIndex > 0 && rbStartOffset === position.column - 1) {\n\t\t\t// edge case, where `position` sits between two tokens belonging to two different languages\n\t\t\tconst [lbStartOffset, lbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(\n\t\t\t\tlineTokens,\n\t\t\t\ttokenIndex - 1\n\t\t\t);\n\t\t\tconst leftBiasedWord = getWordAtText(\n\t\t\t\tposition.column,\n\t\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(),\n\t\t\t\tlineContent.substring(lbStartOffset, lbEndOffset),\n\t\t\t\tlbStartOffset\n\t\t\t);\n\t\t\t// Make sure the result touches the original passed in position\n\t\t\tif (\n\t\t\t\tleftBiasedWord &&\n\t\t\t\tleftBiasedWord.startColumn <= _position.column &&\n\t\t\t\t_position.column <= leftBiasedWord.endColumn\n\t\t\t) {\n\t\t\t\treturn leftBiasedWord;\n\t\t\t}\n\t\t}\n\n\t\treturn null;\n\t}\n\n\tprivate getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {\n\t\treturn this._languageConfigurationService.getLanguageConfiguration(languageId);\n\t}\n\n\tprivate static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {\n\t\tconst languageId = lineTokens.getLanguageId(tokenIndex);\n\n\t\t// go left until a different language is hit\n\t\tlet startOffset = 0;\n\t\tfor (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {\n\t\t\tstartOffset = lineTokens.getStartOffset(i);\n\t\t}\n\n\t\t// go right until a different language is hit\n\t\tlet endOffset = lineTokens.getLineContent().length;\n\t\tfor (\n\t\t\tlet i = tokenIndex, tokenCount = lineTokens.getCount();\n\t\t\ti < tokenCount && lineTokens.getLanguageId(i) === languageId;\n\t\t\ti++\n\t\t) {\n\t\t\tendOffset = lineTokens.getEndOffset(i);\n\t\t}\n\n\t\treturn [startOffset, endOffset];\n\t}\n\n\tpublic getWordUntilPosition(position: IPosition): IWordAtPosition {\n\t\tconst wordAtPosition = this.getWordAtPosition(position);\n\t\tif (!wordAtPosition) {\n\t\t\treturn { word: '', startColumn: position.column, endColumn: position.column, };\n\t\t}\n\t\treturn {\n\t\t\tword: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),\n\t\t\tstartColumn: wordAtPosition.startColumn,\n\t\t\tendColumn: position.column,\n\t\t};\n\t}\n\n\t// #endregion\n\n\t// #region Language Id handling\n\n\tpublic getLanguageId(): string {\n\t\treturn this._languageId;\n\t}\n\n\tpublic getLanguageIdAtPosition(lineNumber: number, column: number): string {\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\treturn lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));\n\t}\n\n\tpublic setLanguageId(languageId: string, source: string = 'api'): void {\n\t\tif (this._languageId === languageId) {\n\t\t\t// There's nothing to do\n\t\t\treturn;\n\t\t}\n\n\t\tconst e: IModelLanguageChangedEvent = {\n\t\t\toldLanguage: this._languageId,\n\t\t\tnewLanguage: languageId,\n\t\t\tsource\n\t\t};\n\n\t\tthis._languageId = languageId;\n\t\tthis._languageIdObs.set(languageId, undefined);\n\t\tthis._bracketPairsTextModelPart.handleDidChangeLanguage(e);\n\n\t\tthis._onDidChangeLanguage.fire(e);\n\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t}\n\n\t// #endregion\n}\n","/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { CharCode } from '../../../../base/common/charCode.js';\nimport { BugIndicatingError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { IPosition, Position } from '../../core/position.js';\nimport { Range } from '../../core/range.js';\nimport { IWordAtPosition, getWordAtText } from '../../core/wordHelper.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { ILanguageService } from '../../languages/language.js';\nimport { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from '../../languages/languageConfigurationRegistry.js';\nimport { BracketPairsTextModelPart } from '../bracketPairsTextModelPart/bracketPairsImpl.js';\nimport { TextModel } from '../textModel.js';\nimport { TextModelPart } from '../textModelPart.js';\nimport { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js';\nimport { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';\nimport { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js';\nimport { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js';\nimport { SparseTokensStore } from '../../tokens/sparseTokensStore.js';\nimport { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';\nimport { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';\nimport { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';\nimport { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';\n\nexport class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {\n\tprivate readonly _semanticTokens: SparseTokensStore;\n\n\tprivate readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent>;\n\tpublic readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;\n\n\tprivate readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent>;\n\tpublic readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;\n\n\tprivate readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;\n\tpublic readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;\n\n\tpublic readonly tokens: IObservable<AbstractSyntaxTokenBackend>;\n\tprivate readonly _useTreeSitter: IObservable<boolean>;\n\tprivate readonly _languageIdObs: ISettableObservable<string>;\n\n\tconstructor(\n\t\tprivate readonly _textModel: TextModel,\n\t\tprivate readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,\n\t\tprivate _languageId: string,\n\t\tprivate readonly _attachedViews: AttachedViews,\n\t\t@ILanguageService private readonly _languageService: ILanguageService,\n\t\t@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t) {\n\t\tsuper();\n\n\t\tthis._languageIdObs = observableValue(this, this._languageId);\n\n\t\tthis._useTreeSitter = derived(this, reader => {\n\t\t\tconst languageId = this._languageIdObs.read(reader);\n\t\t\treturn this._treeSitterLibraryService.supportsLanguage(languageId, reader);\n\t\t});\n\n\t\tthis.tokens = derived(this, reader => {\n\t\t\tlet tokens: AbstractSyntaxTokenBackend;\n\t\t\tif (this._useTreeSitter.read(reader)) {\n\t\t\t\ttokens = reader.store.add(this._instantiationService.createInstance(\n\t\t\t\t\tTreeSitterSyntaxTokenBackend,\n\t\t\t\t\tthis._languageIdObs,\n\t\t\t\t\tthis._languageService.languageIdCodec,\n\t\t\t\t\tthis._textModel,\n\t\t\t\t\tthis._attachedViews.visibleLineRanges\n\t\t\t\t));\n\t\t\t} else {\n\t\t\t\ttokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));\n\t\t\t}\n\n\t\t\treader.store.add(tokens.onDidChangeTokens(e => {\n\t\t\t\tthis._emitModelTokensChangedEvent(e);\n\t\t\t}));\n\n\t\t\treader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {\n\t\t\t\tthis._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();\n\t\t\t}));\n\t\t\treturn tokens;\n\t\t});\n\n\t\tlet hadTokens = false;\n\t\tthis.tokens.recomputeInitiallyAndOnChange(this._store, value => {\n\t\t\tif (hadTokens) {\n\t\t\t\t// We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.\n\t\t\t\t// TODO@hediet: Look into why this is needed.\n\t\t\t\tvalue.todo_resetTokenization();\n\t\t\t}\n\t\t\thadTokens = true;\n\t\t});\n\n\t\tthis._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);\n\t\tthis._onDidChangeLanguage = this._register(new Emitter<IModelLanguageChangedEvent>());\n\t\tthis.onDidChangeLanguage = this._onDidChangeLanguage.event;\n\t\tthis._onDidChangeLanguageConfiguration = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>());\n\t\tthis.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;\n\t\tthis._onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());\n\t\tthis.onDidChangeTokens = this._onDidChangeTokens.event;\n\t}\n\n\tpublic handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {\n\t\tif (e.affects(this._languageId)) {\n\t\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t\t}\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\tthis._semanticTokens.flush();\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._semanticTokens.acceptEdit(\n\t\t\t\t\tc.range,\n\t\t\t\t\teolCount,\n\t\t\t\t\tfirstLineLength,\n\t\t\t\t\tlastLineLength,\n\t\t\t\t\tc.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tthis.tokens.get().handleDidChangeContent(e);\n\t}\n\n\tpublic handleDidChangeAttached(): void {\n\t\tthis.tokens.get().handleDidChangeAttached();\n\t}\n\n\t/**\n\t * Includes grammar and semantic tokens.\n\t */\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tconst syntacticTokens = this.tokens.get().getLineTokens(lineNumber);\n\t\treturn this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);\n\t}\n\n\tprivate _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {\n\t\tif (!this._textModel._isDisposing()) {\n\t\t\tthis._bracketPairsTextModelPart.handleDidChangeTokens(e);\n\t\t\tthis._onDidChangeTokens.fire(e);\n\t\t}\n\t}\n\n\t// #region Grammar Tokens\n\n\tprivate validateLineNumber(lineNumber: number): void {\n\t\tif (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {\n\t\t\tthrow new BugIndicatingError('Illegal value for lineNumber');\n\t\t}\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this.tokens.get().hasTokens;\n\t}\n\n\tpublic resetTokenization() {\n\t\tthis.tokens.get().todo_resetTokenization();\n\t}\n\n\tpublic get backgroundTokenizationState() {\n\t\treturn this.tokens.get().backgroundTokenizationState;\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().forceTokenization(lineNumber);\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic tokenizeIfCheap(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().tokenizeIfCheap(lineNumber);\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\treturn this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);\n\t}\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\treturn this.tokens.get().tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\t// #endregion\n\n\t// #region Semantic Tokens\n\n\tpublic setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {\n\t\tthis._semanticTokens.set(tokens, isComplete, this._textModel);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: tokens !== null,\n\t\t\tranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],\n\t\t});\n\t}\n\n\tpublic hasCompleteSemanticTokens(): boolean {\n\t\treturn this._semanticTokens.isComplete();\n\t}\n\n\tpublic hasSomeSemanticTokens(): boolean {\n\t\treturn !this._semanticTokens.isEmpty();\n\t}\n\n\tpublic setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {\n\t\tif (this.hasCompleteSemanticTokens()) {\n\t\t\treturn;\n\t\t}\n\t\tconst changedRange = this._textModel.validateRange(\n\t\t\tthis._semanticTokens.setPartial(range, tokens)\n\t\t);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: true,\n\t\t\tranges: [\n\t\t\t\t{\n\t\t\t\t\tfromLineNumber: changedRange.startLineNumber,\n\t\t\t\t\ttoLineNumber: changedRange.endLineNumber,\n\t\t\t\t},\n\t\t\t],\n\t\t});\n\t}\n\n\t// #endregion\n\n\t// #region Utility Methods\n\n\tpublic getWordAtPosition(_position: IPosition): IWordAtPosition | null {\n\t\tthis.assertNotDisposed();\n\n\t\tconst position = this._textModel.validatePosition(_position);\n\t\tconst lineContent = this._textModel.getLineContent(position.lineNumber);\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\tconst tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);\n\n\t\t// (1). First try checking right biased word\n\t\tconst [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);\n\t\tconst rightBiasedWord = getWordAtText(\n\t\t\tposition.column,\n\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(),\n\t\t\tlineContent.substring(rbStartOffset, rbEndOffset),\n\t\t\trbStartOffset\n\t\t);\n\t\t// Make sure the result touches the original passed in position\n\t\tif (\n\t\t\trightBiasedWord &&\n\t\t\trightBiasedWord.startColumn <= _position.column &&\n\t\t\t_position.column <= rightBiasedWord.endColumn\n\t\t) {\n\t\t\treturn rightBiasedWord;\n\t\t}\n\n\t\t// (2). Else, if we were at a language boundary, check the left biased word\n\t\tif (tokenIndex > 0 && rbStartOffset === position.column - 1) {\n\t\t\t// edge case, where `position` sits between two tokens belonging to two different languages\n\t\t\tconst [lbStartOffset, lbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(\n\t\t\t\tlineTokens,\n\t\t\t\ttokenIndex - 1\n\t\t\t);\n\t\t\tconst leftBiasedWord = getWordAtText(\n\t\t\t\tposition.column,\n\t\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(),\n\t\t\t\tlineContent.substring(lbStartOffset, lbEndOffset),\n\t\t\t\tlbStartOffset\n\t\t\t);\n\t\t\t// Make sure the result touches the original passed in position\n\t\t\tif (\n\t\t\t\tleftBiasedWord &&\n\t\t\t\tleftBiasedWord.startColumn <= _position.column &&\n\t\t\t\t_position.column <= leftBiasedWord.endColumn\n\t\t\t) {\n\t\t\t\treturn leftBiasedWord;\n\t\t\t}\n\t\t}\n\n\t\treturn null;\n\t}\n\n\tprivate getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {\n\t\treturn this._languageConfigurationService.getLanguageConfiguration(languageId);\n\t}\n\n\tprivate static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {\n\t\tconst languageId = lineTokens.getLanguageId(tokenIndex);\n\n\t\t// go left until a different language is hit\n\t\tlet startOffset = 0;\n\t\tfor (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {\n\t\t\tstartOffset = lineTokens.getStartOffset(i);\n\t\t}\n\n\t\t// go right until a different language is hit\n\t\tlet endOffset = lineTokens.getLineContent().length;\n\t\tfor (\n\t\t\tlet i = tokenIndex, tokenCount = lineTokens.getCount();\n\t\t\ti < tokenCount && lineTokens.getLanguageId(i) === languageId;\n\t\t\ti++\n\t\t) {\n\t\t\tendOffset = lineTokens.getEndOffset(i);\n\t\t}\n\n\t\treturn [startOffset, endOffset];\n\t}\n\n\tpublic getWordUntilPosition(position: IPosition): IWordAtPosition {\n\t\tconst wordAtPosition = this.getWordAtPosition(position);\n\t\tif (!wordAtPosition) {\n\t\t\treturn { word: '', startColumn: position.column, endColumn: position.column, };\n\t\t}\n\t\treturn {\n\t\t\tword: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),\n\t\t\tstartColumn: wordAtPosition.startColumn,\n\t\t\tendColumn: position.column,\n\t\t};\n\t}\n\n\t// #endregion\n\n\t// #region Language Id handling\n\n\tpublic getLanguageId(): string {\n\t\treturn this._languageId;\n\t}\n\n\tpublic getLanguageIdAtPosition(lineNumber: number, column: number): string {\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\treturn lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));\n\t}\n\n\tpublic setLanguageId(languageId: string, source: string = 'api'): void {\n\t\tif (this._languageId === languageId) {\n\t\t\t// There's nothing to do\n\t\t\treturn;\n\t\t}\n\n\t\tconst e: IModelLanguageChangedEvent = {\n\t\t\toldLanguage: this._languageId,\n\t\t\tnewLanguage: languageId,\n\t\t\tsource\n\t\t};\n\n\t\tthis._languageId = languageId;\n\t\tthis._languageIdObs.set(languageId, undefined);\n\t\tthis._bracketPairsTextModelPart.handleDidChangeLanguage(e);\n\n\t\tthis._onDidChangeLanguage.fire(e);\n\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t}\n\n\t// #endregion\n}\n"]}
1
+ {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts","vs/editor/common/model/tokens/tokenizationTextModelPart.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;;;;;;;;;;;AAGhG,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,OAAO,EAAS,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAa,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAmB,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,6BAA6B,EAA0E,MAAM,kDAAkD,CAAC;AAGzK,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,4BAA4B,EAAE,MAAM,8CAA8C,CAAC;AAK5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,qBAAqB,EAAE,MAAM,4DAA4D,CAAC;AACnG,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,uDAAuD,CAAC;AAClG,OAAO,EAAE,OAAO,EAAoC,eAAe,EAAE,MAAM,uCAAuC,CAAC;AAE5G,IAAM,yBAAyB,iCAA/B,MAAM,yBAA0B,SAAQ,aAAa;IAmB3D,YACkB,UAAqB,EACrB,0BAAqD,EAC9D,WAAmB,EACV,cAA6B,EAC5B,gBAAmD,EACtC,6BAA6E,EACrF,qBAA6D,EACzD,yBAAqE;QAEhG,KAAK,EAAE,CAAC;QATS,eAAU,GAAV,UAAU,CAAW;QACrB,+BAA0B,GAA1B,0BAA0B,CAA2B;QAC9D,gBAAW,GAAX,WAAW,CAAQ;QACV,mBAAc,GAAd,cAAc,CAAe;QACX,qBAAgB,GAAhB,gBAAgB,CAAkB;QACrB,kCAA6B,GAA7B,6BAA6B,CAA+B;QACpE,0BAAqB,GAArB,qBAAqB,CAAuB;QACxC,8BAAyB,GAAzB,yBAAyB,CAA2B;QAfhF,2BAAsB,GAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAgC,CAAC,CAAC;QAC7H,0BAAqB,GAAwC,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC;QAkB9G,IAAI,CAAC,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9D,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YACpC,IAAI,MAAkC,CAAC;YACvC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAClE,4BAA4B,EAC5B,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,gBAAgB,CAAC,eAAe,EACrC,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,cAAc,CAAC,iBAAiB,CACrC,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,2BAA2B,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YACjK,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;gBAC7C,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC,CAAC;YACJ,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE;gBACjD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrC,CAAC;YACF,CAAC,CAAC,CAAC,CAAC;YAEJ,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,sCAAsC,CAAC,CAAC,CAAC,EAAE;gBAClE,IAAI,CAAC,0BAA0B,CAAC,0CAA0C,EAAE,CAAC;YAC9E,CAAC,CAAC,CAAC,CAAC;YACJ,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,6BAA6B,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YAC9D,IAAI,SAAS,EAAE,CAAC;gBACf,wJAAwJ;gBACxJ,6CAA6C;gBAC7C,KAAK,CAAC,sBAAsB,EAAE,CAAC;YAChC,CAAC;YACD,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACpF,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA8B,CAAC,CAAC;QACtF,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,iCAAiC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA2C,CAAC,CAAC;QAChH,IAAI,CAAC,gCAAgC,GAAG,IAAI,CAAC,iCAAiC,CAAC,KAAK,CAAC;QACrF,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC;QACvD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAgC,CAAC,CAAC;QAC1F,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC;IAChE,CAAC;IAEM,wCAAwC,CAAC,CAA0C;QACzF,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC;IACF,CAAC;IAEM,sBAAsB,CAAC,CAA4B;QACzD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;aAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,gDAAgD;YAC5E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,cAAc,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAErE,IAAI,CAAC,eAAe,CAAC,UAAU,CAC9B,CAAC,CAAC,KAAK,EACP,QAAQ,EACR,eAAe,EACf,cAAc,EACd,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAc,CACxD,CAAC;YACH,CAAC;QACF,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAEM,uBAAuB;QAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,uBAAuB,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACI,aAAa,CAAC,UAAkB;QACtC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IAC1E,CAAC;IAEO,4BAA4B,CAAC,CAA2B;QAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,yBAAyB;IAEjB,kBAAkB,CAAC,UAAkB;QAC5C,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;QAC9D,CAAC;IACF,CAAC;IAED,IAAW,SAAS;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;IACpC,CAAC;IAEM,iBAAiB;QACvB,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAW,2BAA2B;QACrC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,2BAA2B,CAAC;IACtD,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAEM,wBAAwB,CAAC,UAAkB;QACjD,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAC/D,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACxD,CAAC;IAEM,eAAe,CAAC,UAAkB;QACxC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAEM,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,gCAAgC,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC1F,CAAC;IAEM,eAAe,CAAC,UAAkB,EAAE,KAAe;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC7D,CAAC;IAED,aAAa;IAEb,0BAA0B;IAEnB,iBAAiB,CAAC,MAAsC,EAAE,UAAmB;QACnF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAE9D,IAAI,CAAC,4BAA4B,CAAC;YACjC,qBAAqB,EAAE,MAAM,KAAK,IAAI;YACtC,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;SAC7E,CAAC,CAAC;IACJ,CAAC;IAEM,yBAAyB;QAC/B,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;IAC1C,CAAC;IAEM,qBAAqB;QAC3B,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;IACxC,CAAC;IAEM,wBAAwB,CAAC,KAAY,EAAE,MAA+B;QAC5E,IAAI,IAAI,CAAC,yBAAyB,EAAE,EAAE,CAAC;YACtC,OAAO;QACR,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CACjD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAC9C,CAAC;QAEF,IAAI,CAAC,4BAA4B,CAAC;YACjC,qBAAqB,EAAE,IAAI;YAC3B,MAAM,EAAE;gBACP;oBACC,cAAc,EAAE,YAAY,CAAC,eAAe;oBAC5C,YAAY,EAAE,YAAY,CAAC,aAAa;iBACxC;aACD;SACD,CAAC,CAAC;IACJ,CAAC;IAED,aAAa;IAEb,0BAA0B;IAEnB,iBAAiB,CAAC,SAAoB;QAC5C,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,UAAU,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,2BAAyB,CAAC,uBAAuB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/G,MAAM,eAAe,GAAG,aAAa,CACpC,QAAQ,CAAC,MAAM,EACf,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,iBAAiB,EAAE,EACvF,WAAW,CAAC,SAAS,CAAC,aAAa,EAAE,WAAW,CAAC,EACjD,aAAa,CACb,CAAC;QACF,+DAA+D;QAC/D,IACC,eAAe;YACf,eAAe,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM;YAC/C,SAAS,CAAC,MAAM,IAAI,eAAe,CAAC,SAAS,EAC5C,CAAC;YACF,OAAO,eAAe,CAAC;QACxB,CAAC;QAED,2EAA2E;QAC3E,IAAI,UAAU,GAAG,CAAC,IAAI,aAAa,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7D,2FAA2F;YAC3F,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,2BAAyB,CAAC,uBAAuB,CACrF,UAAU,EACV,UAAU,GAAG,CAAC,CACd,CAAC;YACF,MAAM,cAAc,GAAG,aAAa,CACnC,QAAQ,CAAC,MAAM,EACf,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,EAAE,EAC3F,WAAW,CAAC,SAAS,CAAC,aAAa,EAAE,WAAW,CAAC,EACjD,aAAa,CACb,CAAC;YACF,+DAA+D;YAC/D,IACC,cAAc;gBACd,cAAc,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM;gBAC9C,SAAS,CAAC,MAAM,IAAI,cAAc,CAAC,SAAS,EAC3C,CAAC;gBACF,OAAO,cAAc,CAAC;YACvB,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,wBAAwB,CAAC,UAAkB;QAClD,OAAO,IAAI,CAAC,6BAA6B,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAChF,CAAC;IAEO,MAAM,CAAC,uBAAuB,CAAC,UAAsB,EAAE,UAAkB;QAChF,MAAM,UAAU,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAExD,4CAA4C;QAC5C,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpF,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,6CAA6C;QAC7C,IAAI,SAAS,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC;QACnD,KACC,IAAI,CAAC,GAAG,UAAU,EAAE,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,EACtD,CAAC,GAAG,UAAU,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,UAAU,EAC5D,CAAC,EAAE,EACF,CAAC;YACF,SAAS,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;IAEM,oBAAoB,CAAC,QAAmB;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;QAChF,CAAC;QACD,OAAO;YACN,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC;YACjF,WAAW,EAAE,cAAc,CAAC,WAAW;YACvC,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC1B,CAAC;IACH,CAAC;IAED,aAAa;IAEb,+BAA+B;IAExB,aAAa;QACnB,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAEM,uBAAuB,CAAC,UAAkB,EAAE,MAAc;QAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3D,OAAO,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAEM,aAAa,CAAC,UAAkB,EAAE,SAAiB,KAAK;QAC9D,IAAI,IAAI,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACrC,wBAAwB;YACxB,OAAO;QACR,CAAC;QAED,MAAM,CAAC,GAA+B;YACrC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,WAAW,EAAE,UAAU;YACvB,MAAM;SACN,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,0BAA0B,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;QAE3D,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjD,CAAC;CAGD,CAAA;AA9VY,yBAAyB;IAwBnC,WAAA,gBAAgB,CAAA;IAChB,WAAA,6BAA6B,CAAA;IAC7B,WAAA,qBAAqB,CAAA;IACrB,WAAA,yBAAyB,CAAA;GA3Bf,yBAAyB,CA8VrC","file":"tokenizationTextModelPart.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { CharCode } from '../../../../base/common/charCode.js';\nimport { BugIndicatingError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { IPosition, Position } from '../../core/position.js';\nimport { Range } from '../../core/range.js';\nimport { IWordAtPosition, getWordAtText } from '../../core/wordHelper.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { ILanguageService } from '../../languages/language.js';\nimport { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from '../../languages/languageConfigurationRegistry.js';\nimport { BracketPairsTextModelPart } from '../bracketPairsTextModelPart/bracketPairsImpl.js';\nimport { TextModel } from '../textModel.js';\nimport { TextModelPart } from '../textModelPart.js';\nimport { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js';\nimport { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';\nimport { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js';\nimport { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js';\nimport { SparseTokensStore } from '../../tokens/sparseTokensStore.js';\nimport { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';\nimport { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';\nimport { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';\nimport { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';\n\nexport class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {\n\tprivate readonly _semanticTokens: SparseTokensStore;\n\n\tprivate readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent>;\n\tpublic readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;\n\n\tprivate readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent>;\n\tpublic readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;\n\n\tprivate readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;\n\tpublic readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;\n\n\tprivate readonly _onDidChangeFontTokens: Emitter<IModelFontTokensChangedEvent> = this._register(new Emitter<IModelFontTokensChangedEvent>());\n\tpublic readonly onDidChangeFontTokens: Event<IModelFontTokensChangedEvent> = this._onDidChangeFontTokens.event;\n\n\tpublic readonly tokens: IObservable<AbstractSyntaxTokenBackend>;\n\tprivate readonly _useTreeSitter: IObservable<boolean>;\n\tprivate readonly _languageIdObs: ISettableObservable<string>;\n\n\tconstructor(\n\t\tprivate readonly _textModel: TextModel,\n\t\tprivate readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,\n\t\tprivate _languageId: string,\n\t\tprivate readonly _attachedViews: AttachedViews,\n\t\t@ILanguageService private readonly _languageService: ILanguageService,\n\t\t@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t) {\n\t\tsuper();\n\n\t\tthis._languageIdObs = observableValue(this, this._languageId);\n\n\t\tthis._useTreeSitter = derived(this, reader => {\n\t\t\tconst languageId = this._languageIdObs.read(reader);\n\t\t\treturn this._treeSitterLibraryService.supportsLanguage(languageId, reader);\n\t\t});\n\n\t\tthis.tokens = derived(this, reader => {\n\t\t\tlet tokens: AbstractSyntaxTokenBackend;\n\t\t\tif (this._useTreeSitter.read(reader)) {\n\t\t\t\ttokens = reader.store.add(this._instantiationService.createInstance(\n\t\t\t\t\tTreeSitterSyntaxTokenBackend,\n\t\t\t\t\tthis._languageIdObs,\n\t\t\t\t\tthis._languageService.languageIdCodec,\n\t\t\t\t\tthis._textModel,\n\t\t\t\t\tthis._attachedViews.visibleLineRanges\n\t\t\t\t));\n\t\t\t} else {\n\t\t\t\ttokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));\n\t\t\t}\n\n\t\t\treader.store.add(tokens.onDidChangeTokens(e => {\n\t\t\t\tthis._emitModelTokensChangedEvent(e);\n\t\t\t}));\n\t\t\treader.store.add(tokens.onDidChangeFontTokens(e => {\n\t\t\t\tif (!this._textModel._isDisposing()) {\n\t\t\t\t\tthis._onDidChangeFontTokens.fire(e);\n\t\t\t\t}\n\t\t\t}));\n\n\t\t\treader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {\n\t\t\t\tthis._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();\n\t\t\t}));\n\t\t\treturn tokens;\n\t\t});\n\n\t\tlet hadTokens = false;\n\t\tthis.tokens.recomputeInitiallyAndOnChange(this._store, value => {\n\t\t\tif (hadTokens) {\n\t\t\t\t// We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.\n\t\t\t\t// TODO@hediet: Look into why this is needed.\n\t\t\t\tvalue.todo_resetTokenization();\n\t\t\t}\n\t\t\thadTokens = true;\n\t\t});\n\n\t\tthis._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);\n\t\tthis._onDidChangeLanguage = this._register(new Emitter<IModelLanguageChangedEvent>());\n\t\tthis.onDidChangeLanguage = this._onDidChangeLanguage.event;\n\t\tthis._onDidChangeLanguageConfiguration = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>());\n\t\tthis.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;\n\t\tthis._onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());\n\t\tthis.onDidChangeTokens = this._onDidChangeTokens.event;\n\t\tthis._onDidChangeFontTokens = this._register(new Emitter<IModelFontTokensChangedEvent>());\n\t\tthis.onDidChangeFontTokens = this._onDidChangeFontTokens.event;\n\t}\n\n\tpublic handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {\n\t\tif (e.affects(this._languageId)) {\n\t\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t\t}\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\tthis._semanticTokens.flush();\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._semanticTokens.acceptEdit(\n\t\t\t\t\tc.range,\n\t\t\t\t\teolCount,\n\t\t\t\t\tfirstLineLength,\n\t\t\t\t\tlastLineLength,\n\t\t\t\t\tc.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tthis.tokens.get().handleDidChangeContent(e);\n\t}\n\n\tpublic handleDidChangeAttached(): void {\n\t\tthis.tokens.get().handleDidChangeAttached();\n\t}\n\n\t/**\n\t * Includes grammar and semantic tokens.\n\t */\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tconst syntacticTokens = this.tokens.get().getLineTokens(lineNumber);\n\t\treturn this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);\n\t}\n\n\tprivate _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {\n\t\tif (!this._textModel._isDisposing()) {\n\t\t\tthis._bracketPairsTextModelPart.handleDidChangeTokens(e);\n\t\t\tthis._onDidChangeTokens.fire(e);\n\t\t}\n\t}\n\n\t// #region Grammar Tokens\n\n\tprivate validateLineNumber(lineNumber: number): void {\n\t\tif (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {\n\t\t\tthrow new BugIndicatingError('Illegal value for lineNumber');\n\t\t}\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this.tokens.get().hasTokens;\n\t}\n\n\tpublic resetTokenization() {\n\t\tthis.tokens.get().todo_resetTokenization();\n\t}\n\n\tpublic get backgroundTokenizationState() {\n\t\treturn this.tokens.get().backgroundTokenizationState;\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().forceTokenization(lineNumber);\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic tokenizeIfCheap(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().tokenizeIfCheap(lineNumber);\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\treturn this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);\n\t}\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\treturn this.tokens.get().tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\t// #endregion\n\n\t// #region Semantic Tokens\n\n\tpublic setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {\n\t\tthis._semanticTokens.set(tokens, isComplete, this._textModel);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: tokens !== null,\n\t\t\tranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],\n\t\t});\n\t}\n\n\tpublic hasCompleteSemanticTokens(): boolean {\n\t\treturn this._semanticTokens.isComplete();\n\t}\n\n\tpublic hasSomeSemanticTokens(): boolean {\n\t\treturn !this._semanticTokens.isEmpty();\n\t}\n\n\tpublic setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {\n\t\tif (this.hasCompleteSemanticTokens()) {\n\t\t\treturn;\n\t\t}\n\t\tconst changedRange = this._textModel.validateRange(\n\t\t\tthis._semanticTokens.setPartial(range, tokens)\n\t\t);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: true,\n\t\t\tranges: [\n\t\t\t\t{\n\t\t\t\t\tfromLineNumber: changedRange.startLineNumber,\n\t\t\t\t\ttoLineNumber: changedRange.endLineNumber,\n\t\t\t\t},\n\t\t\t],\n\t\t});\n\t}\n\n\t// #endregion\n\n\t// #region Utility Methods\n\n\tpublic getWordAtPosition(_position: IPosition): IWordAtPosition | null {\n\t\tthis.assertNotDisposed();\n\n\t\tconst position = this._textModel.validatePosition(_position);\n\t\tconst lineContent = this._textModel.getLineContent(position.lineNumber);\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\tconst tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);\n\n\t\t// (1). First try checking right biased word\n\t\tconst [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);\n\t\tconst rightBiasedWord = getWordAtText(\n\t\t\tposition.column,\n\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(),\n\t\t\tlineContent.substring(rbStartOffset, rbEndOffset),\n\t\t\trbStartOffset\n\t\t);\n\t\t// Make sure the result touches the original passed in position\n\t\tif (\n\t\t\trightBiasedWord &&\n\t\t\trightBiasedWord.startColumn <= _position.column &&\n\t\t\t_position.column <= rightBiasedWord.endColumn\n\t\t) {\n\t\t\treturn rightBiasedWord;\n\t\t}\n\n\t\t// (2). Else, if we were at a language boundary, check the left biased word\n\t\tif (tokenIndex > 0 && rbStartOffset === position.column - 1) {\n\t\t\t// edge case, where `position` sits between two tokens belonging to two different languages\n\t\t\tconst [lbStartOffset, lbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(\n\t\t\t\tlineTokens,\n\t\t\t\ttokenIndex - 1\n\t\t\t);\n\t\t\tconst leftBiasedWord = getWordAtText(\n\t\t\t\tposition.column,\n\t\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(),\n\t\t\t\tlineContent.substring(lbStartOffset, lbEndOffset),\n\t\t\t\tlbStartOffset\n\t\t\t);\n\t\t\t// Make sure the result touches the original passed in position\n\t\t\tif (\n\t\t\t\tleftBiasedWord &&\n\t\t\t\tleftBiasedWord.startColumn <= _position.column &&\n\t\t\t\t_position.column <= leftBiasedWord.endColumn\n\t\t\t) {\n\t\t\t\treturn leftBiasedWord;\n\t\t\t}\n\t\t}\n\n\t\treturn null;\n\t}\n\n\tprivate getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {\n\t\treturn this._languageConfigurationService.getLanguageConfiguration(languageId);\n\t}\n\n\tprivate static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {\n\t\tconst languageId = lineTokens.getLanguageId(tokenIndex);\n\n\t\t// go left until a different language is hit\n\t\tlet startOffset = 0;\n\t\tfor (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {\n\t\t\tstartOffset = lineTokens.getStartOffset(i);\n\t\t}\n\n\t\t// go right until a different language is hit\n\t\tlet endOffset = lineTokens.getLineContent().length;\n\t\tfor (\n\t\t\tlet i = tokenIndex, tokenCount = lineTokens.getCount();\n\t\t\ti < tokenCount && lineTokens.getLanguageId(i) === languageId;\n\t\t\ti++\n\t\t) {\n\t\t\tendOffset = lineTokens.getEndOffset(i);\n\t\t}\n\n\t\treturn [startOffset, endOffset];\n\t}\n\n\tpublic getWordUntilPosition(position: IPosition): IWordAtPosition {\n\t\tconst wordAtPosition = this.getWordAtPosition(position);\n\t\tif (!wordAtPosition) {\n\t\t\treturn { word: '', startColumn: position.column, endColumn: position.column, };\n\t\t}\n\t\treturn {\n\t\t\tword: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),\n\t\t\tstartColumn: wordAtPosition.startColumn,\n\t\t\tendColumn: position.column,\n\t\t};\n\t}\n\n\t// #endregion\n\n\t// #region Language Id handling\n\n\tpublic getLanguageId(): string {\n\t\treturn this._languageId;\n\t}\n\n\tpublic getLanguageIdAtPosition(lineNumber: number, column: number): string {\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\treturn lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));\n\t}\n\n\tpublic setLanguageId(languageId: string, source: string = 'api'): void {\n\t\tif (this._languageId === languageId) {\n\t\t\t// There's nothing to do\n\t\t\treturn;\n\t\t}\n\n\t\tconst e: IModelLanguageChangedEvent = {\n\t\t\toldLanguage: this._languageId,\n\t\t\tnewLanguage: languageId,\n\t\t\tsource\n\t\t};\n\n\t\tthis._languageId = languageId;\n\t\tthis._languageIdObs.set(languageId, undefined);\n\t\tthis._bracketPairsTextModelPart.handleDidChangeLanguage(e);\n\n\t\tthis._onDidChangeLanguage.fire(e);\n\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t}\n\n\t// #endregion\n}\n","/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { CharCode } from '../../../../base/common/charCode.js';\nimport { BugIndicatingError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { IPosition, Position } from '../../core/position.js';\nimport { Range } from '../../core/range.js';\nimport { IWordAtPosition, getWordAtText } from '../../core/wordHelper.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { ILanguageService } from '../../languages/language.js';\nimport { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from '../../languages/languageConfigurationRegistry.js';\nimport { BracketPairsTextModelPart } from '../bracketPairsTextModelPart/bracketPairsImpl.js';\nimport { TextModel } from '../textModel.js';\nimport { TextModelPart } from '../textModelPart.js';\nimport { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js';\nimport { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';\nimport { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js';\nimport { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js';\nimport { SparseTokensStore } from '../../tokens/sparseTokensStore.js';\nimport { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';\nimport { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';\nimport { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';\nimport { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';\n\nexport class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {\n\tprivate readonly _semanticTokens: SparseTokensStore;\n\n\tprivate readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent>;\n\tpublic readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;\n\n\tprivate readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent>;\n\tpublic readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;\n\n\tprivate readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;\n\tpublic readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;\n\n\tprivate readonly _onDidChangeFontTokens: Emitter<IModelFontTokensChangedEvent> = this._register(new Emitter<IModelFontTokensChangedEvent>());\n\tpublic readonly onDidChangeFontTokens: Event<IModelFontTokensChangedEvent> = this._onDidChangeFontTokens.event;\n\n\tpublic readonly tokens: IObservable<AbstractSyntaxTokenBackend>;\n\tprivate readonly _useTreeSitter: IObservable<boolean>;\n\tprivate readonly _languageIdObs: ISettableObservable<string>;\n\n\tconstructor(\n\t\tprivate readonly _textModel: TextModel,\n\t\tprivate readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,\n\t\tprivate _languageId: string,\n\t\tprivate readonly _attachedViews: AttachedViews,\n\t\t@ILanguageService private readonly _languageService: ILanguageService,\n\t\t@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t) {\n\t\tsuper();\n\n\t\tthis._languageIdObs = observableValue(this, this._languageId);\n\n\t\tthis._useTreeSitter = derived(this, reader => {\n\t\t\tconst languageId = this._languageIdObs.read(reader);\n\t\t\treturn this._treeSitterLibraryService.supportsLanguage(languageId, reader);\n\t\t});\n\n\t\tthis.tokens = derived(this, reader => {\n\t\t\tlet tokens: AbstractSyntaxTokenBackend;\n\t\t\tif (this._useTreeSitter.read(reader)) {\n\t\t\t\ttokens = reader.store.add(this._instantiationService.createInstance(\n\t\t\t\t\tTreeSitterSyntaxTokenBackend,\n\t\t\t\t\tthis._languageIdObs,\n\t\t\t\t\tthis._languageService.languageIdCodec,\n\t\t\t\t\tthis._textModel,\n\t\t\t\t\tthis._attachedViews.visibleLineRanges\n\t\t\t\t));\n\t\t\t} else {\n\t\t\t\ttokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));\n\t\t\t}\n\n\t\t\treader.store.add(tokens.onDidChangeTokens(e => {\n\t\t\t\tthis._emitModelTokensChangedEvent(e);\n\t\t\t}));\n\t\t\treader.store.add(tokens.onDidChangeFontTokens(e => {\n\t\t\t\tif (!this._textModel._isDisposing()) {\n\t\t\t\t\tthis._onDidChangeFontTokens.fire(e);\n\t\t\t\t}\n\t\t\t}));\n\n\t\t\treader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {\n\t\t\t\tthis._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();\n\t\t\t}));\n\t\t\treturn tokens;\n\t\t});\n\n\t\tlet hadTokens = false;\n\t\tthis.tokens.recomputeInitiallyAndOnChange(this._store, value => {\n\t\t\tif (hadTokens) {\n\t\t\t\t// We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.\n\t\t\t\t// TODO@hediet: Look into why this is needed.\n\t\t\t\tvalue.todo_resetTokenization();\n\t\t\t}\n\t\t\thadTokens = true;\n\t\t});\n\n\t\tthis._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);\n\t\tthis._onDidChangeLanguage = this._register(new Emitter<IModelLanguageChangedEvent>());\n\t\tthis.onDidChangeLanguage = this._onDidChangeLanguage.event;\n\t\tthis._onDidChangeLanguageConfiguration = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>());\n\t\tthis.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;\n\t\tthis._onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());\n\t\tthis.onDidChangeTokens = this._onDidChangeTokens.event;\n\t\tthis._onDidChangeFontTokens = this._register(new Emitter<IModelFontTokensChangedEvent>());\n\t\tthis.onDidChangeFontTokens = this._onDidChangeFontTokens.event;\n\t}\n\n\tpublic handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {\n\t\tif (e.affects(this._languageId)) {\n\t\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t\t}\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\tthis._semanticTokens.flush();\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._semanticTokens.acceptEdit(\n\t\t\t\t\tc.range,\n\t\t\t\t\teolCount,\n\t\t\t\t\tfirstLineLength,\n\t\t\t\t\tlastLineLength,\n\t\t\t\t\tc.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tthis.tokens.get().handleDidChangeContent(e);\n\t}\n\n\tpublic handleDidChangeAttached(): void {\n\t\tthis.tokens.get().handleDidChangeAttached();\n\t}\n\n\t/**\n\t * Includes grammar and semantic tokens.\n\t */\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tconst syntacticTokens = this.tokens.get().getLineTokens(lineNumber);\n\t\treturn this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);\n\t}\n\n\tprivate _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {\n\t\tif (!this._textModel._isDisposing()) {\n\t\t\tthis._bracketPairsTextModelPart.handleDidChangeTokens(e);\n\t\t\tthis._onDidChangeTokens.fire(e);\n\t\t}\n\t}\n\n\t// #region Grammar Tokens\n\n\tprivate validateLineNumber(lineNumber: number): void {\n\t\tif (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {\n\t\t\tthrow new BugIndicatingError('Illegal value for lineNumber');\n\t\t}\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this.tokens.get().hasTokens;\n\t}\n\n\tpublic resetTokenization() {\n\t\tthis.tokens.get().todo_resetTokenization();\n\t}\n\n\tpublic get backgroundTokenizationState() {\n\t\treturn this.tokens.get().backgroundTokenizationState;\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().forceTokenization(lineNumber);\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tthis.validateLineNumber(lineNumber);\n\t\treturn this.tokens.get().isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic tokenizeIfCheap(lineNumber: number): void {\n\t\tthis.validateLineNumber(lineNumber);\n\t\tthis.tokens.get().tokenizeIfCheap(lineNumber);\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\treturn this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);\n\t}\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\treturn this.tokens.get().tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\t// #endregion\n\n\t// #region Semantic Tokens\n\n\tpublic setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {\n\t\tthis._semanticTokens.set(tokens, isComplete, this._textModel);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: tokens !== null,\n\t\t\tranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],\n\t\t});\n\t}\n\n\tpublic hasCompleteSemanticTokens(): boolean {\n\t\treturn this._semanticTokens.isComplete();\n\t}\n\n\tpublic hasSomeSemanticTokens(): boolean {\n\t\treturn !this._semanticTokens.isEmpty();\n\t}\n\n\tpublic setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {\n\t\tif (this.hasCompleteSemanticTokens()) {\n\t\t\treturn;\n\t\t}\n\t\tconst changedRange = this._textModel.validateRange(\n\t\t\tthis._semanticTokens.setPartial(range, tokens)\n\t\t);\n\n\t\tthis._emitModelTokensChangedEvent({\n\t\t\tsemanticTokensApplied: true,\n\t\t\tranges: [\n\t\t\t\t{\n\t\t\t\t\tfromLineNumber: changedRange.startLineNumber,\n\t\t\t\t\ttoLineNumber: changedRange.endLineNumber,\n\t\t\t\t},\n\t\t\t],\n\t\t});\n\t}\n\n\t// #endregion\n\n\t// #region Utility Methods\n\n\tpublic getWordAtPosition(_position: IPosition): IWordAtPosition | null {\n\t\tthis.assertNotDisposed();\n\n\t\tconst position = this._textModel.validatePosition(_position);\n\t\tconst lineContent = this._textModel.getLineContent(position.lineNumber);\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\tconst tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);\n\n\t\t// (1). First try checking right biased word\n\t\tconst [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);\n\t\tconst rightBiasedWord = getWordAtText(\n\t\t\tposition.column,\n\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(),\n\t\t\tlineContent.substring(rbStartOffset, rbEndOffset),\n\t\t\trbStartOffset\n\t\t);\n\t\t// Make sure the result touches the original passed in position\n\t\tif (\n\t\t\trightBiasedWord &&\n\t\t\trightBiasedWord.startColumn <= _position.column &&\n\t\t\t_position.column <= rightBiasedWord.endColumn\n\t\t) {\n\t\t\treturn rightBiasedWord;\n\t\t}\n\n\t\t// (2). Else, if we were at a language boundary, check the left biased word\n\t\tif (tokenIndex > 0 && rbStartOffset === position.column - 1) {\n\t\t\t// edge case, where `position` sits between two tokens belonging to two different languages\n\t\t\tconst [lbStartOffset, lbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(\n\t\t\t\tlineTokens,\n\t\t\t\ttokenIndex - 1\n\t\t\t);\n\t\t\tconst leftBiasedWord = getWordAtText(\n\t\t\t\tposition.column,\n\t\t\t\tthis.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(),\n\t\t\t\tlineContent.substring(lbStartOffset, lbEndOffset),\n\t\t\t\tlbStartOffset\n\t\t\t);\n\t\t\t// Make sure the result touches the original passed in position\n\t\t\tif (\n\t\t\t\tleftBiasedWord &&\n\t\t\t\tleftBiasedWord.startColumn <= _position.column &&\n\t\t\t\t_position.column <= leftBiasedWord.endColumn\n\t\t\t) {\n\t\t\t\treturn leftBiasedWord;\n\t\t\t}\n\t\t}\n\n\t\treturn null;\n\t}\n\n\tprivate getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {\n\t\treturn this._languageConfigurationService.getLanguageConfiguration(languageId);\n\t}\n\n\tprivate static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {\n\t\tconst languageId = lineTokens.getLanguageId(tokenIndex);\n\n\t\t// go left until a different language is hit\n\t\tlet startOffset = 0;\n\t\tfor (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {\n\t\t\tstartOffset = lineTokens.getStartOffset(i);\n\t\t}\n\n\t\t// go right until a different language is hit\n\t\tlet endOffset = lineTokens.getLineContent().length;\n\t\tfor (\n\t\t\tlet i = tokenIndex, tokenCount = lineTokens.getCount();\n\t\t\ti < tokenCount && lineTokens.getLanguageId(i) === languageId;\n\t\t\ti++\n\t\t) {\n\t\t\tendOffset = lineTokens.getEndOffset(i);\n\t\t}\n\n\t\treturn [startOffset, endOffset];\n\t}\n\n\tpublic getWordUntilPosition(position: IPosition): IWordAtPosition {\n\t\tconst wordAtPosition = this.getWordAtPosition(position);\n\t\tif (!wordAtPosition) {\n\t\t\treturn { word: '', startColumn: position.column, endColumn: position.column, };\n\t\t}\n\t\treturn {\n\t\t\tword: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),\n\t\t\tstartColumn: wordAtPosition.startColumn,\n\t\t\tendColumn: position.column,\n\t\t};\n\t}\n\n\t// #endregion\n\n\t// #region Language Id handling\n\n\tpublic getLanguageId(): string {\n\t\treturn this._languageId;\n\t}\n\n\tpublic getLanguageIdAtPosition(lineNumber: number, column: number): string {\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tconst lineTokens = this.getLineTokens(position.lineNumber);\n\t\treturn lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));\n\t}\n\n\tpublic setLanguageId(languageId: string, source: string = 'api'): void {\n\t\tif (this._languageId === languageId) {\n\t\t\t// There's nothing to do\n\t\t\treturn;\n\t\t}\n\n\t\tconst e: IModelLanguageChangedEvent = {\n\t\t\toldLanguage: this._languageId,\n\t\t\tnewLanguage: languageId,\n\t\t\tsource\n\t\t};\n\n\t\tthis._languageId = languageId;\n\t\tthis._languageIdObs.set(languageId, undefined);\n\t\tthis._bracketPairsTextModelPart.handleDidChangeLanguage(e);\n\n\t\tthis._onDidChangeLanguage.fire(e);\n\t\tthis._onDidChangeLanguageConfiguration.fire({});\n\t}\n\n\t// #endregion\n}\n"]}
package/esm/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.js CHANGED
@@ -98,6 +98,9 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
98
98
  setTokens: (tokens) => {
99
99
  this.setTokens(tokens);
100
100
  },
101
+ setFontInfo: (changes) => {
102
+ this.setFontInfo(changes);
103
+ },
101
104
  backgroundTokenizationFinished: () => {
102
105
  if (this._backgroundTokenizationState === 2 /* BackgroundTokenizationState.Completed */) {
103
106
  // We already did a full tokenization and don't go back to progressing.
@@ -134,6 +137,9 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
134
137
  setTokens: (tokens) => {
135
138
  this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
136
139
  },
140
+ setFontInfo: (changes) => {
141
+ this.setFontInfo(changes);
142
+ },
137
143
  backgroundTokenizationFinished() {
138
144
  // NO OP
139
145
  },
@@ -178,6 +184,9 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
178
184
  }
179
185
  return { changes: changes };
180
186
  }
187
+ setFontInfo(changes) {
188
+ this._onDidChangeFontTokens.fire({ changes });
189
+ }
181
190
  refreshAllVisibleLineTokens() {
182
191
  const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));
183
192
  this.refreshRanges(ranges);
package/esm/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts","vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;AAEhG,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAS,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACxF,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE3D,OAAO,EAAkD,oBAAoB,EAAsD,MAAM,oBAAoB,CAAC;AAK9J,OAAO,EAAE,gCAAgC,EAAE,MAAM,kDAAkD,CAAC;AACpG,OAAO,EAAE,qBAAqB,EAAE,MAAM,uCAAuC,CAAC;AAG9E,OAAO,EAAE,mCAAmC,EAAE,0BAA0B,EAAE,8BAA8B,EAAE,MAAM,uBAAuB,CAAC;AACxI,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAiB,MAAM,iCAAiC,CAAC;AAEjH,mBAAmB;AACnB,MAAM,OAAO,2BAA4B,SAAQ,0BAA0B;IAiB1E,YACC,eAAiC,EACjC,SAAoB,EACH,aAA2B,EAC5C,aAA4B;QAE5B,KAAK,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QAHjB,kBAAa,GAAb,aAAa,CAAc;QAnBrC,eAAU,GAA+C,IAAI,CAAC;QAC5D,iCAA4B,kDAAuE;QAC1F,4CAAuC,GAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAQ,CAAC,CAAC;QAChG,2CAAsC,GAAgB,IAAI,CAAC,uCAAuC,CAAC,KAAK,CAAC;QAEjH,gCAA2B,GAAsC,IAAI,CAAC;QAC7D,yBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAwB,CAAC,CAAC;QAErF,YAAO,GAAG,IAAI,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAI3D,8BAAyB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAwB,CAAC,CAAC;QAE1F,wBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,aAAa,EAAsC,CAAC,CAAC;QAU9G,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YACrD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACxC,IAAI,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACnD,OAAO;YACR,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;YACzE,IAAI,KAAK,EAAE,CAAC;gBACX,IAAI,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,mBAAmB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAS,CAAC,UAAU,CAAC,CAAC,CAAC;oBACnF,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAC9C,CAAC;gBACD,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,sBAAsB,CAAC,uBAAgC,IAAI;QACjE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,sBAAsB,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,IAAI,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC5B,qBAAqB,EAAE,KAAK;gBAC5B,MAAM,EAAE;oBACP;wBACC,cAAc,EAAE,CAAC;wBACjB,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;qBAC5C;iBACD;aACD,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,sBAAsB,GAAG,GAAkD,EAAE;YAClF,IAAI,IAAI,CAAC,UAAU,CAAC,yBAAyB,EAAE,EAAE,CAAC;gBACjD,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;YAC3E,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,IAAI,YAAoB,CAAC;YACzB,IAAI,CAAC;gBACJ,YAAY,GAAG,mBAAmB,CAAC,eAAe,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,iBAAiB,CAAC,CAAC,CAAC,CAAC;gBACrB,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,OAAO,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;QAC5C,CAAC,CAAC;QAEF,MAAM,CAAC,mBAAmB,EAAE,YAAY,CAAC,GAAG,sBAAsB,EAAE,CAAC;QACrE,IAAI,mBAAmB,IAAI,YAAY,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,mCAAmC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,mBAAmB,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACxJ,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;QAElC,IAAI,CAAC,2BAA2B,GAAG,IAAI,CAAC;QACxC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,CAAC,GAAiC;gBACvC,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;oBACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACxB,CAAC;gBACD,8BAA8B,EAAE,GAAG,EAAE;oBACpC,IAAI,IAAI,CAAC,4BAA4B,kDAA0C,EAAE,CAAC;wBACjF,uEAAuE;wBACvE,OAAO;oBACR,CAAC;oBACD,MAAM,QAAQ,gDAAwC,CAAC;oBACvD,IAAI,CAAC,4BAA4B,GAAG,QAAQ,CAAC;oBAC7C,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,CAAC;gBACrD,CAAC;gBACD,WAAW,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;oBAClC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAAC,OAAO;oBAAC,CAAC;oBACjC,MAAM,8BAA8B,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC;oBACjG,wFAAwF;oBACxF,IAAI,8BAA8B,KAAK,IAAI,IAAI,UAAU,IAAI,8BAA8B,EAAE,CAAC;wBAC7F,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC;aACD,CAAC;YAEF,IAAI,mBAAmB,IAAI,mBAAmB,CAAC,yBAAyB,IAAI,CAAC,mBAAmB,CAAC,yCAAyC,EAAE,CAAC;gBAC5I,IAAI,CAAC,oBAAoB,CAAC,KAAK,GAAG,mBAAmB,CAAC,yBAAyB,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACrG,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,yBAAyB,EAAE,EAAE,CAAC;gBACtF,IAAI,CAAC,oBAAoB,CAAC,KAAK,GAAG,IAAI,CAAC,2BAA2B;oBACjE,IAAI,0BAA0B,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACpD,IAAI,CAAC,2BAA2B,CAAC,aAAa,EAAE,CAAC;YAClD,CAAC;YAED,IAAI,mBAAmB,EAAE,yCAAyC,IAAI,mBAAmB,CAAC,yBAAyB,EAAE,CAAC;gBACrH,IAAI,CAAC,sBAAsB,GAAG,IAAI,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAC/E,IAAI,CAAC,sBAAsB,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;gBACjG,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,CAAC,yBAAyB,CAAC,KAAK,GAAG,mBAAmB,CAAC,yBAAyB,CAAC,IAAI,CAAC,UAAU,EAAE;oBACrG,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;wBACrB,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC1E,CAAC;oBACD,8BAA8B;wBAC7B,QAAQ;oBACT,CAAC;oBACD,WAAW,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;wBAClC,IAAI,CAAC,sBAAsB,EAAE,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBAC7D,CAAC;iBACD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;gBACxC,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;gBACxC,IAAI,CAAC,yBAAyB,CAAC,KAAK,GAAG,SAAS,CAAC;YAClD,CAAC;QACF,CAAC;QAED,IAAI,CAAC,2BAA2B,EAAE,CAAC;IACpC,CAAC;IAEM,uBAAuB;QAC7B,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,sBAAsB,CAAC,CAA4B;QACzD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,iFAAiF;YACjF,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,gDAAgD;YAC5E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAErD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;gBAC5D,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;YAC7E,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAEtD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;QACnD,CAAC;IACF,CAAC;IAEO,SAAS,CAAC,MAAmC;QACpD,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAE7E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IAEO,2BAA2B;QAClC,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC/F,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAEO,aAAa,CAAC,MAA4B;QACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;IACF,CAAC;IAEO,YAAY,CAAC,eAAuB,EAAE,aAAqB;QAClE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO;QACR,CAAC;QAED,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QACzF,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;QAExE,MAAM,OAAO,GAAG,IAAI,gCAAgC,EAAE,CAAC;QACvD,MAAM,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC;QAC3G,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEzD,IAAI,eAAe,EAAE,CAAC;YACrB,uDAAuD;YACvD,oEAAoE;YACpE,yEAAyE;YACzE,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;gBACvC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;YACtF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,MAAM,OAAO,GAAG,IAAI,gCAAgC,EAAE,CAAC;QACvD,IAAI,CAAC,UAAU,EAAE,qBAAqB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,wBAAwB,CAAC,UAAkB;QACjD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAC7D,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,aAAa,CAAC,UAAkB;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CACpC,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,EAC/B,UAAU,GAAG,CAAC,EACd,QAAQ,CACR,CAAC;QACF,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnF,IAAI,IAAI,CAAC,sBAAsB,CAAC,sCAAsC,EAAE,GAAG,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,UAAU,EAAE,CAAC;gBACtK,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAC7D,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,EAC/B,UAAU,GAAG,CAAC,EACd,QAAQ,CACR,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,uBAAuB,EAAE,CAAC;oBACvG,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC;gBAC1E,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAEM,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QAC5F,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,uCAA+B;QAChC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,gCAAgC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC9E,CAAC;IAGM,eAAe,CAAC,UAAkB,EAAE,KAAe;QACzD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED,IAAW,SAAS;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAC/B,CAAC;CACD","file":"tokenizerSyntaxTokenBackend.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { onUnexpectedError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { Position } from '../../core/position.js';\nimport { LineRange } from '../../core/ranges/lineRange.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';\nimport { IAttachedView } from '../../model.js';\nimport { IModelContentChangedEvent } from '../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';\nimport { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';\nimport { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';\nimport { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { TextModel } from '../textModel.js';\nimport { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';\nimport { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';\n\n/** For TextMate */\nexport class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprivate _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;\n\tprivate readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);\n\tprivate _debugBackgroundTokens: ContiguousTokensStore | undefined;\n\tprivate _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;\n\n\tprivate readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());\n\n\tconstructor(\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tprivate readonly getLanguageId: () => string,\n\t\tattachedViews: AttachedViews,\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\t\tthis._register(TokenizationRegistry.onDidChange((e) => {\n\t\t\tconst languageId = this.getLanguageId();\n\t\t\tif (e.changedLanguages.indexOf(languageId) === -1) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tthis.todo_resetTokenization();\n\t\t}));\n\n\t\tthis.todo_resetTokenization();\n\n\t\tthis._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {\n\t\t\tif (state) {\n\t\t\t\tlet existing = this._attachedViewStates.get(view);\n\t\t\t\tif (!existing) {\n\t\t\t\t\texisting = new AttachedViewHandler(() => this.refreshRanges(existing!.lineRanges));\n\t\t\t\t\tthis._attachedViewStates.set(view, existing);\n\t\t\t\t}\n\t\t\t\texisting.handleStateChange(state);\n\t\t\t} else {\n\t\t\t\tthis._attachedViewStates.deleteAndDispose(view);\n\t\t\t}\n\t\t}));\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tthis._tokens.flush();\n\t\tthis._debugBackgroundTokens?.flush();\n\t\tif (this._debugBackgroundStates) {\n\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t}\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\n\t\tconst initializeTokenization = (): [ITokenizationSupport, IState] | [null, null] => {\n\t\t\tif (this._textModel.isTooLargeForTokenization()) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tconst tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());\n\t\t\tif (!tokenizationSupport) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tlet initialState: IState;\n\t\t\ttry {\n\t\t\t\tinitialState = tokenizationSupport.getInitialState();\n\t\t\t} catch (e) {\n\t\t\t\tonUnexpectedError(e);\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\treturn [tokenizationSupport, initialState];\n\t\t};\n\n\t\tconst [tokenizationSupport, initialState] = initializeTokenization();\n\t\tif (tokenizationSupport && initialState) {\n\t\t\tthis._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);\n\t\t} else {\n\t\t\tthis._tokenizer = null;\n\t\t}\n\n\t\tthis._backgroundTokenizer.clear();\n\n\t\tthis._defaultBackgroundTokenizer = null;\n\t\tif (this._tokenizer) {\n\t\t\tconst b: IBackgroundTokenizationStore = {\n\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\tthis.setTokens(tokens);\n\t\t\t\t},\n\t\t\t\tbackgroundTokenizationFinished: () => {\n\t\t\t\t\tif (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {\n\t\t\t\t\t\t// We already did a full tokenization and don't go back to progressing.\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tconst newState = BackgroundTokenizationState.Completed;\n\t\t\t\t\tthis._backgroundTokenizationState = newState;\n\t\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t\t},\n\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\tif (!this._tokenizer) { return; }\n\t\t\t\t\tconst firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();\n\t\t\t\t\t// Don't accept states for definitely valid states, the renderer is ahead of the worker!\n\t\t\t\t\tif (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {\n\t\t\t\t\t\tthis._tokenizer?.store.setEndState(lineNumber, state);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t};\n\n\t\t\tif (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {\n\t\t\t\tthis._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);\n\t\t\t}\n\t\t\tif (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {\n\t\t\t\tthis._backgroundTokenizer.value = this._defaultBackgroundTokenizer =\n\t\t\t\t\tnew DefaultBackgroundTokenizer(this._tokenizer, b);\n\t\t\t\tthis._defaultBackgroundTokenizer.handleChanges();\n\t\t\t}\n\n\t\t\tif (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {\n\t\t\t\tthis._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);\n\t\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t\t\tthis._debugBackgroundTokenizer.clear();\n\t\t\t\tthis._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {\n\t\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\t\tthis._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);\n\t\t\t\t\t},\n\t\t\t\t\tbackgroundTokenizationFinished() {\n\t\t\t\t\t\t// NO OP\n\t\t\t\t\t},\n\t\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\t\tthis._debugBackgroundStates?.setEndState(lineNumber, state);\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tthis._debugBackgroundTokens = undefined;\n\t\t\t\tthis._debugBackgroundStates = undefined;\n\t\t\t\tthis._debugBackgroundTokenizer.value = undefined;\n\t\t\t}\n\t\t}\n\n\t\tthis.refreshAllVisibleLineTokens();\n\t}\n\n\tpublic handleDidChangeAttached() {\n\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._tokens.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t\tthis._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t}\n\t\t\tthis._debugBackgroundStates?.acceptChanges(e.changes);\n\n\t\t\tif (this._tokenizer) {\n\t\t\t\tthis._tokenizer.store.acceptChanges(e.changes);\n\t\t\t}\n\t\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t\t}\n\t}\n\n\tprivate setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {\n\t\tconst { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);\n\n\t\tif (changes.length > 0) {\n\t\t\tthis._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });\n\t\t}\n\n\t\treturn { changes: changes };\n\t}\n\n\tprivate refreshAllVisibleLineTokens(): void {\n\t\tconst ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));\n\t\tthis.refreshRanges(ranges);\n\t}\n\n\tprivate refreshRanges(ranges: readonly LineRange[]): void {\n\t\tfor (const range of ranges) {\n\t\t\tthis.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);\n\t\t}\n\t}\n\n\tprivate refreshRange(startLineNumber: number, endLineNumber: number): void {\n\t\tif (!this._tokenizer) {\n\t\t\treturn;\n\t\t}\n\n\t\tstartLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));\n\t\tendLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);\n\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tconst { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);\n\t\tconst changedTokens = this.setTokens(builder.finalize());\n\n\t\tif (heuristicTokens) {\n\t\t\t// We overrode tokens with heuristically computed ones.\n\t\t\t// Because old states might get reused (thus stopping invalidation),\n\t\t\t// we have to explicitly request the tokens for the changed ranges again.\n\t\t\tfor (const c of changedTokens.changes) {\n\t\t\t\tthis._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);\n\t\t\t}\n\t\t}\n\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tthis._tokenizer?.updateTokensUntilLine(builder, lineNumber);\n\t\tthis.setTokens(builder.finalize());\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst lineText = this._textModel.getLineContent(lineNumber);\n\t\tconst result = this._tokens.getTokens(\n\t\t\tthis._textModel.getLanguageId(),\n\t\t\tlineNumber - 1,\n\t\t\tlineText\n\t\t);\n\t\tif (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {\n\t\t\tif (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {\n\t\t\t\tconst backgroundResult = this._debugBackgroundTokens.getTokens(\n\t\t\t\t\tthis._textModel.getLanguageId(),\n\t\t\t\t\tlineNumber - 1,\n\t\t\t\t\tlineText\n\t\t\t\t);\n\t\t\t\tif (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {\n\t\t\t\t\tthis._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn result;\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\tif (!this._tokenizer) {\n\t\t\treturn StandardTokenType.Other;\n\t\t}\n\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tthis.forceTokenization(position.lineNumber);\n\t\treturn this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);\n\t}\n\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tif (!this._tokenizer) {\n\t\t\treturn null;\n\t\t}\n\t\tthis.forceTokenization(lineNumber);\n\t\treturn this._tokenizer.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this._tokens.hasTokens;\n\t}\n}\n","/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { onUnexpectedError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { Position } from '../../core/position.js';\nimport { LineRange } from '../../core/ranges/lineRange.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';\nimport { IAttachedView } from '../../model.js';\nimport { IModelContentChangedEvent } from '../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';\nimport { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';\nimport { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';\nimport { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { TextModel } from '../textModel.js';\nimport { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';\nimport { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';\n\n/** For TextMate */\nexport class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprivate _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;\n\tprivate readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);\n\tprivate _debugBackgroundTokens: ContiguousTokensStore | undefined;\n\tprivate _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;\n\n\tprivate readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());\n\n\tconstructor(\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tprivate readonly getLanguageId: () => string,\n\t\tattachedViews: AttachedViews,\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\t\tthis._register(TokenizationRegistry.onDidChange((e) => {\n\t\t\tconst languageId = this.getLanguageId();\n\t\t\tif (e.changedLanguages.indexOf(languageId) === -1) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tthis.todo_resetTokenization();\n\t\t}));\n\n\t\tthis.todo_resetTokenization();\n\n\t\tthis._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {\n\t\t\tif (state) {\n\t\t\t\tlet existing = this._attachedViewStates.get(view);\n\t\t\t\tif (!existing) {\n\t\t\t\t\texisting = new AttachedViewHandler(() => this.refreshRanges(existing!.lineRanges));\n\t\t\t\t\tthis._attachedViewStates.set(view, existing);\n\t\t\t\t}\n\t\t\t\texisting.handleStateChange(state);\n\t\t\t} else {\n\t\t\t\tthis._attachedViewStates.deleteAndDispose(view);\n\t\t\t}\n\t\t}));\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tthis._tokens.flush();\n\t\tthis._debugBackgroundTokens?.flush();\n\t\tif (this._debugBackgroundStates) {\n\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t}\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\n\t\tconst initializeTokenization = (): [ITokenizationSupport, IState] | [null, null] => {\n\t\t\tif (this._textModel.isTooLargeForTokenization()) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tconst tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());\n\t\t\tif (!tokenizationSupport) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tlet initialState: IState;\n\t\t\ttry {\n\t\t\t\tinitialState = tokenizationSupport.getInitialState();\n\t\t\t} catch (e) {\n\t\t\t\tonUnexpectedError(e);\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\treturn [tokenizationSupport, initialState];\n\t\t};\n\n\t\tconst [tokenizationSupport, initialState] = initializeTokenization();\n\t\tif (tokenizationSupport && initialState) {\n\t\t\tthis._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);\n\t\t} else {\n\t\t\tthis._tokenizer = null;\n\t\t}\n\n\t\tthis._backgroundTokenizer.clear();\n\n\t\tthis._defaultBackgroundTokenizer = null;\n\t\tif (this._tokenizer) {\n\t\t\tconst b: IBackgroundTokenizationStore = {\n\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\tthis.setTokens(tokens);\n\t\t\t\t},\n\t\t\t\tbackgroundTokenizationFinished: () => {\n\t\t\t\t\tif (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {\n\t\t\t\t\t\t// We already did a full tokenization and don't go back to progressing.\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tconst newState = BackgroundTokenizationState.Completed;\n\t\t\t\t\tthis._backgroundTokenizationState = newState;\n\t\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t\t},\n\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\tif (!this._tokenizer) { return; }\n\t\t\t\t\tconst firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();\n\t\t\t\t\t// Don't accept states for definitely valid states, the renderer is ahead of the worker!\n\t\t\t\t\tif (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {\n\t\t\t\t\t\tthis._tokenizer?.store.setEndState(lineNumber, state);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t};\n\n\t\t\tif (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {\n\t\t\t\tthis._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);\n\t\t\t}\n\t\t\tif (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {\n\t\t\t\tthis._backgroundTokenizer.value = this._defaultBackgroundTokenizer =\n\t\t\t\t\tnew DefaultBackgroundTokenizer(this._tokenizer, b);\n\t\t\t\tthis._defaultBackgroundTokenizer.handleChanges();\n\t\t\t}\n\n\t\t\tif (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {\n\t\t\t\tthis._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);\n\t\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t\t\tthis._debugBackgroundTokenizer.clear();\n\t\t\t\tthis._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {\n\t\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\t\tthis._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);\n\t\t\t\t\t},\n\t\t\t\t\tbackgroundTokenizationFinished() {\n\t\t\t\t\t\t// NO OP\n\t\t\t\t\t},\n\t\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\t\tthis._debugBackgroundStates?.setEndState(lineNumber, state);\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tthis._debugBackgroundTokens = undefined;\n\t\t\t\tthis._debugBackgroundStates = undefined;\n\t\t\t\tthis._debugBackgroundTokenizer.value = undefined;\n\t\t\t}\n\t\t}\n\n\t\tthis.refreshAllVisibleLineTokens();\n\t}\n\n\tpublic handleDidChangeAttached() {\n\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._tokens.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t\tthis._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t}\n\t\t\tthis._debugBackgroundStates?.acceptChanges(e.changes);\n\n\t\t\tif (this._tokenizer) {\n\t\t\t\tthis._tokenizer.store.acceptChanges(e.changes);\n\t\t\t}\n\t\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t\t}\n\t}\n\n\tprivate setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {\n\t\tconst { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);\n\n\t\tif (changes.length > 0) {\n\t\t\tthis._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });\n\t\t}\n\n\t\treturn { changes: changes };\n\t}\n\n\tprivate refreshAllVisibleLineTokens(): void {\n\t\tconst ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));\n\t\tthis.refreshRanges(ranges);\n\t}\n\n\tprivate refreshRanges(ranges: readonly LineRange[]): void {\n\t\tfor (const range of ranges) {\n\t\t\tthis.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);\n\t\t}\n\t}\n\n\tprivate refreshRange(startLineNumber: number, endLineNumber: number): void {\n\t\tif (!this._tokenizer) {\n\t\t\treturn;\n\t\t}\n\n\t\tstartLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));\n\t\tendLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);\n\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tconst { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);\n\t\tconst changedTokens = this.setTokens(builder.finalize());\n\n\t\tif (heuristicTokens) {\n\t\t\t// We overrode tokens with heuristically computed ones.\n\t\t\t// Because old states might get reused (thus stopping invalidation),\n\t\t\t// we have to explicitly request the tokens for the changed ranges again.\n\t\t\tfor (const c of changedTokens.changes) {\n\t\t\t\tthis._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);\n\t\t\t}\n\t\t}\n\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tthis._tokenizer?.updateTokensUntilLine(builder, lineNumber);\n\t\tthis.setTokens(builder.finalize());\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst lineText = this._textModel.getLineContent(lineNumber);\n\t\tconst result = this._tokens.getTokens(\n\t\t\tthis._textModel.getLanguageId(),\n\t\t\tlineNumber - 1,\n\t\t\tlineText\n\t\t);\n\t\tif (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {\n\t\t\tif (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {\n\t\t\t\tconst backgroundResult = this._debugBackgroundTokens.getTokens(\n\t\t\t\t\tthis._textModel.getLanguageId(),\n\t\t\t\t\tlineNumber - 1,\n\t\t\t\t\tlineText\n\t\t\t\t);\n\t\t\t\tif (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {\n\t\t\t\t\tthis._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn result;\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\tif (!this._tokenizer) {\n\t\t\treturn StandardTokenType.Other;\n\t\t}\n\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tthis.forceTokenization(position.lineNumber);\n\t\treturn this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);\n\t}\n\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tif (!this._tokenizer) {\n\t\t\treturn null;\n\t\t}\n\t\tthis.forceTokenization(lineNumber);\n\t\treturn this._tokenizer.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this._tokens.hasTokens;\n\t}\n}\n"]}
1
+ {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts","vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;AAEhG,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAS,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACxF,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE3D,OAAO,EAAkD,oBAAoB,EAAsD,MAAM,oBAAoB,CAAC;AAK9J,OAAO,EAAE,gCAAgC,EAAE,MAAM,kDAAkD,CAAC;AACpG,OAAO,EAAE,qBAAqB,EAAE,MAAM,uCAAuC,CAAC;AAG9E,OAAO,EAAE,mCAAmC,EAAE,0BAA0B,EAAE,8BAA8B,EAAE,MAAM,uBAAuB,CAAC;AACxI,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAiB,MAAM,iCAAiC,CAAC;AAEjH,mBAAmB;AACnB,MAAM,OAAO,2BAA4B,SAAQ,0BAA0B;IAiB1E,YACC,eAAiC,EACjC,SAAoB,EACH,aAA2B,EAC5C,aAA4B;QAE5B,KAAK,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QAHjB,kBAAa,GAAb,aAAa,CAAc;QAnBrC,eAAU,GAA+C,IAAI,CAAC;QAC5D,iCAA4B,kDAAuE;QAC1F,4CAAuC,GAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAQ,CAAC,CAAC;QAChG,2CAAsC,GAAgB,IAAI,CAAC,uCAAuC,CAAC,KAAK,CAAC;QAEjH,gCAA2B,GAAsC,IAAI,CAAC;QAC7D,yBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAwB,CAAC,CAAC;QAErF,YAAO,GAAG,IAAI,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAI3D,8BAAyB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,iBAAiB,EAAwB,CAAC,CAAC;QAE1F,wBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,aAAa,EAAsC,CAAC,CAAC;QAU9G,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;YACrD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACxC,IAAI,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACnD,OAAO;YACR,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,wBAAwB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;YACzE,IAAI,KAAK,EAAE,CAAC;gBACX,IAAI,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,mBAAmB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAS,CAAC,UAAU,CAAC,CAAC,CAAC;oBACnF,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAC9C,CAAC;gBACD,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,sBAAsB,CAAC,uBAAgC,IAAI;QACjE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,sBAAsB,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACjC,IAAI,CAAC,sBAAsB,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,IAAI,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC5B,qBAAqB,EAAE,KAAK;gBAC5B,MAAM,EAAE;oBACP;wBACC,cAAc,EAAE,CAAC;wBACjB,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;qBAC5C;iBACD;aACD,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,sBAAsB,GAAG,GAAkD,EAAE;YAClF,IAAI,IAAI,CAAC,UAAU,CAAC,yBAAyB,EAAE,EAAE,CAAC;gBACjD,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;YAC3E,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,IAAI,YAAoB,CAAC;YACzB,IAAI,CAAC;gBACJ,YAAY,GAAG,mBAAmB,CAAC,eAAe,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,iBAAiB,CAAC,CAAC,CAAC,CAAC;gBACrB,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,OAAO,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;QAC5C,CAAC,CAAC;QAEF,MAAM,CAAC,mBAAmB,EAAE,YAAY,CAAC,GAAG,sBAAsB,EAAE,CAAC;QACrE,IAAI,mBAAmB,IAAI,YAAY,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,mCAAmC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,mBAAmB,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACxJ,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;QAElC,IAAI,CAAC,2BAA2B,GAAG,IAAI,CAAC;QACxC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,CAAC,GAAiC;gBACvC,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;oBACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACxB,CAAC;gBACD,WAAW,EAAE,CAAC,OAAyB,EAAE,EAAE;oBAC1C,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;gBACD,8BAA8B,EAAE,GAAG,EAAE;oBACpC,IAAI,IAAI,CAAC,4BAA4B,kDAA0C,EAAE,CAAC;wBACjF,uEAAuE;wBACvE,OAAO;oBACR,CAAC;oBACD,MAAM,QAAQ,gDAAwC,CAAC;oBACvD,IAAI,CAAC,4BAA4B,GAAG,QAAQ,CAAC;oBAC7C,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,CAAC;gBACrD,CAAC;gBACD,WAAW,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;oBAClC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAAC,OAAO;oBAAC,CAAC;oBACjC,MAAM,8BAA8B,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC;oBACjG,wFAAwF;oBACxF,IAAI,8BAA8B,KAAK,IAAI,IAAI,UAAU,IAAI,8BAA8B,EAAE,CAAC;wBAC7F,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC;aACD,CAAC;YAEF,IAAI,mBAAmB,IAAI,mBAAmB,CAAC,yBAAyB,IAAI,CAAC,mBAAmB,CAAC,yCAAyC,EAAE,CAAC;gBAC5I,IAAI,CAAC,oBAAoB,CAAC,KAAK,GAAG,mBAAmB,CAAC,yBAAyB,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACrG,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,yBAAyB,EAAE,EAAE,CAAC;gBACtF,IAAI,CAAC,oBAAoB,CAAC,KAAK,GAAG,IAAI,CAAC,2BAA2B;oBACjE,IAAI,0BAA0B,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACpD,IAAI,CAAC,2BAA2B,CAAC,aAAa,EAAE,CAAC;YAClD,CAAC;YAED,IAAI,mBAAmB,EAAE,yCAAyC,IAAI,mBAAmB,CAAC,yBAAyB,EAAE,CAAC;gBACrH,IAAI,CAAC,sBAAsB,GAAG,IAAI,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAC/E,IAAI,CAAC,sBAAsB,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;gBACjG,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,CAAC,yBAAyB,CAAC,KAAK,GAAG,mBAAmB,CAAC,yBAAyB,CAAC,IAAI,CAAC,UAAU,EAAE;oBACrG,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;wBACrB,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC1E,CAAC;oBACD,WAAW,EAAE,CAAC,OAAyB,EAAE,EAAE;wBAC1C,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;oBACD,8BAA8B;wBAC7B,QAAQ;oBACT,CAAC;oBACD,WAAW,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;wBAClC,IAAI,CAAC,sBAAsB,EAAE,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBAC7D,CAAC;iBACD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;gBACxC,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;gBACxC,IAAI,CAAC,yBAAyB,CAAC,KAAK,GAAG,SAAS,CAAC;YAClD,CAAC;QACF,CAAC;QAED,IAAI,CAAC,2BAA2B,EAAE,CAAC;IACpC,CAAC;IAEM,uBAAuB;QAC7B,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,sBAAsB,CAAC,CAA4B;QACzD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,iFAAiF;YACjF,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,gDAAgD;YAC5E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC3B,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAErD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;gBAC5D,IAAI,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;YAC7E,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAEtD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;QACnD,CAAC;IACF,CAAC;IAEO,SAAS,CAAC,MAAmC;QACpD,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAE7E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,qBAAqB,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IAEO,WAAW,CAAC,OAAyB;QAC5C,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/C,CAAC;IAEO,2BAA2B;QAClC,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAC/F,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAEO,aAAa,CAAC,MAA4B;QACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;IACF,CAAC;IAEO,YAAY,CAAC,eAAuB,EAAE,aAAqB;QAClE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO;QACR,CAAC;QAED,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QACzF,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;QAExE,MAAM,OAAO,GAAG,IAAI,gCAAgC,EAAE,CAAC;QACvD,MAAM,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC;QAC3G,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEzD,IAAI,eAAe,EAAE,CAAC;YACrB,uDAAuD;YACvD,oEAAoE;YACpE,yEAAyE;YACzE,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;gBACvC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;YACtF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,MAAM,OAAO,GAAG,IAAI,gCAAgC,EAAE,CAAC;QACvD,IAAI,CAAC,UAAU,EAAE,qBAAqB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE,CAAC;IACnD,CAAC;IAEM,wBAAwB,CAAC,UAAkB;QACjD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAC7D,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QAC1C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,aAAa,CAAC,UAAkB;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CACpC,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,EAC/B,UAAU,GAAG,CAAC,EACd,QAAQ,CACR,CAAC;QACF,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnF,IAAI,IAAI,CAAC,sBAAsB,CAAC,sCAAsC,EAAE,GAAG,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,UAAU,EAAE,CAAC;gBACtK,MAAM,gBAAgB,GAAG,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAC7D,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,EAC/B,UAAU,GAAG,CAAC,EACd,QAAQ,CACR,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,uBAAuB,EAAE,CAAC;oBACvG,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC;gBAC1E,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAEM,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QAC5F,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,uCAA+B;QAChC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,gCAAgC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC9E,CAAC;IAGM,eAAe,CAAC,UAAkB,EAAE,KAAe;QACzD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED,IAAW,SAAS;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IAC/B,CAAC;CACD","file":"tokenizerSyntaxTokenBackend.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { onUnexpectedError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { Position } from '../../core/position.js';\nimport { LineRange } from '../../core/ranges/lineRange.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';\nimport { IAttachedView } from '../../model.js';\nimport { FontTokensUpdate, IModelContentChangedEvent } from '../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';\nimport { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';\nimport { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';\nimport { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { TextModel } from '../textModel.js';\nimport { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';\nimport { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';\n\n/** For TextMate */\nexport class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprivate _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;\n\tprivate readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);\n\tprivate _debugBackgroundTokens: ContiguousTokensStore | undefined;\n\tprivate _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;\n\n\tprivate readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());\n\n\tconstructor(\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tprivate readonly getLanguageId: () => string,\n\t\tattachedViews: AttachedViews,\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\t\tthis._register(TokenizationRegistry.onDidChange((e) => {\n\t\t\tconst languageId = this.getLanguageId();\n\t\t\tif (e.changedLanguages.indexOf(languageId) === -1) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tthis.todo_resetTokenization();\n\t\t}));\n\n\t\tthis.todo_resetTokenization();\n\n\t\tthis._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {\n\t\t\tif (state) {\n\t\t\t\tlet existing = this._attachedViewStates.get(view);\n\t\t\t\tif (!existing) {\n\t\t\t\t\texisting = new AttachedViewHandler(() => this.refreshRanges(existing!.lineRanges));\n\t\t\t\t\tthis._attachedViewStates.set(view, existing);\n\t\t\t\t}\n\t\t\t\texisting.handleStateChange(state);\n\t\t\t} else {\n\t\t\t\tthis._attachedViewStates.deleteAndDispose(view);\n\t\t\t}\n\t\t}));\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tthis._tokens.flush();\n\t\tthis._debugBackgroundTokens?.flush();\n\t\tif (this._debugBackgroundStates) {\n\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t}\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\n\t\tconst initializeTokenization = (): [ITokenizationSupport, IState] | [null, null] => {\n\t\t\tif (this._textModel.isTooLargeForTokenization()) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tconst tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());\n\t\t\tif (!tokenizationSupport) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tlet initialState: IState;\n\t\t\ttry {\n\t\t\t\tinitialState = tokenizationSupport.getInitialState();\n\t\t\t} catch (e) {\n\t\t\t\tonUnexpectedError(e);\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\treturn [tokenizationSupport, initialState];\n\t\t};\n\n\t\tconst [tokenizationSupport, initialState] = initializeTokenization();\n\t\tif (tokenizationSupport && initialState) {\n\t\t\tthis._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);\n\t\t} else {\n\t\t\tthis._tokenizer = null;\n\t\t}\n\n\t\tthis._backgroundTokenizer.clear();\n\n\t\tthis._defaultBackgroundTokenizer = null;\n\t\tif (this._tokenizer) {\n\t\t\tconst b: IBackgroundTokenizationStore = {\n\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\tthis.setTokens(tokens);\n\t\t\t\t},\n\t\t\t\tsetFontInfo: (changes: FontTokensUpdate) => {\n\t\t\t\t\tthis.setFontInfo(changes);\n\t\t\t\t},\n\t\t\t\tbackgroundTokenizationFinished: () => {\n\t\t\t\t\tif (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {\n\t\t\t\t\t\t// We already did a full tokenization and don't go back to progressing.\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tconst newState = BackgroundTokenizationState.Completed;\n\t\t\t\t\tthis._backgroundTokenizationState = newState;\n\t\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t\t},\n\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\tif (!this._tokenizer) { return; }\n\t\t\t\t\tconst firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();\n\t\t\t\t\t// Don't accept states for definitely valid states, the renderer is ahead of the worker!\n\t\t\t\t\tif (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {\n\t\t\t\t\t\tthis._tokenizer?.store.setEndState(lineNumber, state);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t};\n\n\t\t\tif (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {\n\t\t\t\tthis._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);\n\t\t\t}\n\t\t\tif (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {\n\t\t\t\tthis._backgroundTokenizer.value = this._defaultBackgroundTokenizer =\n\t\t\t\t\tnew DefaultBackgroundTokenizer(this._tokenizer, b);\n\t\t\t\tthis._defaultBackgroundTokenizer.handleChanges();\n\t\t\t}\n\n\t\t\tif (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {\n\t\t\t\tthis._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);\n\t\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t\t\tthis._debugBackgroundTokenizer.clear();\n\t\t\t\tthis._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {\n\t\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\t\tthis._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);\n\t\t\t\t\t},\n\t\t\t\t\tsetFontInfo: (changes: FontTokensUpdate) => {\n\t\t\t\t\t\tthis.setFontInfo(changes);\n\t\t\t\t\t},\n\t\t\t\t\tbackgroundTokenizationFinished() {\n\t\t\t\t\t\t// NO OP\n\t\t\t\t\t},\n\t\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\t\tthis._debugBackgroundStates?.setEndState(lineNumber, state);\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tthis._debugBackgroundTokens = undefined;\n\t\t\t\tthis._debugBackgroundStates = undefined;\n\t\t\t\tthis._debugBackgroundTokenizer.value = undefined;\n\t\t\t}\n\t\t}\n\n\t\tthis.refreshAllVisibleLineTokens();\n\t}\n\n\tpublic handleDidChangeAttached() {\n\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._tokens.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t\tthis._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t}\n\t\t\tthis._debugBackgroundStates?.acceptChanges(e.changes);\n\n\t\t\tif (this._tokenizer) {\n\t\t\t\tthis._tokenizer.store.acceptChanges(e.changes);\n\t\t\t}\n\t\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t\t}\n\t}\n\n\tprivate setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {\n\t\tconst { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);\n\n\t\tif (changes.length > 0) {\n\t\t\tthis._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });\n\t\t}\n\n\t\treturn { changes: changes };\n\t}\n\n\tprivate setFontInfo(changes: FontTokensUpdate): void {\n\t\tthis._onDidChangeFontTokens.fire({ changes });\n\t}\n\n\tprivate refreshAllVisibleLineTokens(): void {\n\t\tconst ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));\n\t\tthis.refreshRanges(ranges);\n\t}\n\n\tprivate refreshRanges(ranges: readonly LineRange[]): void {\n\t\tfor (const range of ranges) {\n\t\t\tthis.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);\n\t\t}\n\t}\n\n\tprivate refreshRange(startLineNumber: number, endLineNumber: number): void {\n\t\tif (!this._tokenizer) {\n\t\t\treturn;\n\t\t}\n\n\t\tstartLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));\n\t\tendLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);\n\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tconst { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);\n\t\tconst changedTokens = this.setTokens(builder.finalize());\n\n\t\tif (heuristicTokens) {\n\t\t\t// We overrode tokens with heuristically computed ones.\n\t\t\t// Because old states might get reused (thus stopping invalidation),\n\t\t\t// we have to explicitly request the tokens for the changed ranges again.\n\t\t\tfor (const c of changedTokens.changes) {\n\t\t\t\tthis._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);\n\t\t\t}\n\t\t}\n\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tthis._tokenizer?.updateTokensUntilLine(builder, lineNumber);\n\t\tthis.setTokens(builder.finalize());\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst lineText = this._textModel.getLineContent(lineNumber);\n\t\tconst result = this._tokens.getTokens(\n\t\t\tthis._textModel.getLanguageId(),\n\t\t\tlineNumber - 1,\n\t\t\tlineText\n\t\t);\n\t\tif (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {\n\t\t\tif (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {\n\t\t\t\tconst backgroundResult = this._debugBackgroundTokens.getTokens(\n\t\t\t\t\tthis._textModel.getLanguageId(),\n\t\t\t\t\tlineNumber - 1,\n\t\t\t\t\tlineText\n\t\t\t\t);\n\t\t\t\tif (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {\n\t\t\t\t\tthis._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn result;\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\tif (!this._tokenizer) {\n\t\t\treturn StandardTokenType.Other;\n\t\t}\n\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tthis.forceTokenization(position.lineNumber);\n\t\treturn this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);\n\t}\n\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tif (!this._tokenizer) {\n\t\t\treturn null;\n\t\t}\n\t\tthis.forceTokenization(lineNumber);\n\t\treturn this._tokenizer.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this._tokens.hasTokens;\n\t}\n}\n","/*---------------------------------------------------------------------------------------------\n * Copyright (c) Microsoft Corporation. All rights reserved.\n * Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { onUnexpectedError } from '../../../../base/common/errors.js';\nimport { Emitter, Event } from '../../../../base/common/event.js';\nimport { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';\nimport { countEOL } from '../../core/misc/eolCounter.js';\nimport { Position } from '../../core/position.js';\nimport { LineRange } from '../../core/ranges/lineRange.js';\nimport { StandardTokenType } from '../../encodedTokenAttributes.js';\nimport { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';\nimport { IAttachedView } from '../../model.js';\nimport { FontTokensUpdate, IModelContentChangedEvent } from '../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';\nimport { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';\nimport { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';\nimport { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';\nimport { LineTokens } from '../../tokens/lineTokens.js';\nimport { TextModel } from '../textModel.js';\nimport { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';\nimport { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';\n\n/** For TextMate */\nexport class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprivate _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;\n\tprivate readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);\n\tprivate _debugBackgroundTokens: ContiguousTokensStore | undefined;\n\tprivate _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;\n\n\tprivate readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());\n\n\tprivate readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());\n\n\tconstructor(\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tprivate readonly getLanguageId: () => string,\n\t\tattachedViews: AttachedViews,\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\t\tthis._register(TokenizationRegistry.onDidChange((e) => {\n\t\t\tconst languageId = this.getLanguageId();\n\t\t\tif (e.changedLanguages.indexOf(languageId) === -1) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tthis.todo_resetTokenization();\n\t\t}));\n\n\t\tthis.todo_resetTokenization();\n\n\t\tthis._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {\n\t\t\tif (state) {\n\t\t\t\tlet existing = this._attachedViewStates.get(view);\n\t\t\t\tif (!existing) {\n\t\t\t\t\texisting = new AttachedViewHandler(() => this.refreshRanges(existing!.lineRanges));\n\t\t\t\t\tthis._attachedViewStates.set(view, existing);\n\t\t\t\t}\n\t\t\t\texisting.handleStateChange(state);\n\t\t\t} else {\n\t\t\t\tthis._attachedViewStates.deleteAndDispose(view);\n\t\t\t}\n\t\t}));\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tthis._tokens.flush();\n\t\tthis._debugBackgroundTokens?.flush();\n\t\tif (this._debugBackgroundStates) {\n\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t}\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\n\t\tconst initializeTokenization = (): [ITokenizationSupport, IState] | [null, null] => {\n\t\t\tif (this._textModel.isTooLargeForTokenization()) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tconst tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());\n\t\t\tif (!tokenizationSupport) {\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\tlet initialState: IState;\n\t\t\ttry {\n\t\t\t\tinitialState = tokenizationSupport.getInitialState();\n\t\t\t} catch (e) {\n\t\t\t\tonUnexpectedError(e);\n\t\t\t\treturn [null, null];\n\t\t\t}\n\t\t\treturn [tokenizationSupport, initialState];\n\t\t};\n\n\t\tconst [tokenizationSupport, initialState] = initializeTokenization();\n\t\tif (tokenizationSupport && initialState) {\n\t\t\tthis._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);\n\t\t} else {\n\t\t\tthis._tokenizer = null;\n\t\t}\n\n\t\tthis._backgroundTokenizer.clear();\n\n\t\tthis._defaultBackgroundTokenizer = null;\n\t\tif (this._tokenizer) {\n\t\t\tconst b: IBackgroundTokenizationStore = {\n\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\tthis.setTokens(tokens);\n\t\t\t\t},\n\t\t\t\tsetFontInfo: (changes: FontTokensUpdate) => {\n\t\t\t\t\tthis.setFontInfo(changes);\n\t\t\t\t},\n\t\t\t\tbackgroundTokenizationFinished: () => {\n\t\t\t\t\tif (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {\n\t\t\t\t\t\t// We already did a full tokenization and don't go back to progressing.\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tconst newState = BackgroundTokenizationState.Completed;\n\t\t\t\t\tthis._backgroundTokenizationState = newState;\n\t\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t\t},\n\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\tif (!this._tokenizer) { return; }\n\t\t\t\t\tconst firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();\n\t\t\t\t\t// Don't accept states for definitely valid states, the renderer is ahead of the worker!\n\t\t\t\t\tif (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {\n\t\t\t\t\t\tthis._tokenizer?.store.setEndState(lineNumber, state);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t};\n\n\t\t\tif (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {\n\t\t\t\tthis._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);\n\t\t\t}\n\t\t\tif (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {\n\t\t\t\tthis._backgroundTokenizer.value = this._defaultBackgroundTokenizer =\n\t\t\t\t\tnew DefaultBackgroundTokenizer(this._tokenizer, b);\n\t\t\t\tthis._defaultBackgroundTokenizer.handleChanges();\n\t\t\t}\n\n\t\t\tif (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {\n\t\t\t\tthis._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);\n\t\t\t\tthis._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());\n\t\t\t\tthis._debugBackgroundTokenizer.clear();\n\t\t\t\tthis._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {\n\t\t\t\t\tsetTokens: (tokens) => {\n\t\t\t\t\t\tthis._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);\n\t\t\t\t\t},\n\t\t\t\t\tsetFontInfo: (changes: FontTokensUpdate) => {\n\t\t\t\t\t\tthis.setFontInfo(changes);\n\t\t\t\t\t},\n\t\t\t\t\tbackgroundTokenizationFinished() {\n\t\t\t\t\t\t// NO OP\n\t\t\t\t\t},\n\t\t\t\t\tsetEndState: (lineNumber, state) => {\n\t\t\t\t\t\tthis._debugBackgroundStates?.setEndState(lineNumber, state);\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tthis._debugBackgroundTokens = undefined;\n\t\t\t\tthis._debugBackgroundStates = undefined;\n\t\t\t\tthis._debugBackgroundTokenizer.value = undefined;\n\t\t\t}\n\t\t}\n\n\t\tthis.refreshAllVisibleLineTokens();\n\t}\n\n\tpublic handleDidChangeAttached() {\n\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t}\n\n\tpublic handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else if (!e.isEolChange) { // We don't have to do anything on an EOL change\n\t\t\tfor (const c of e.changes) {\n\t\t\t\tconst [eolCount, firstLineLength] = countEOL(c.text);\n\n\t\t\t\tthis._tokens.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t\tthis._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);\n\t\t\t}\n\t\t\tthis._debugBackgroundStates?.acceptChanges(e.changes);\n\n\t\t\tif (this._tokenizer) {\n\t\t\t\tthis._tokenizer.store.acceptChanges(e.changes);\n\t\t\t}\n\t\t\tthis._defaultBackgroundTokenizer?.handleChanges();\n\t\t}\n\t}\n\n\tprivate setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {\n\t\tconst { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);\n\n\t\tif (changes.length > 0) {\n\t\t\tthis._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });\n\t\t}\n\n\t\treturn { changes: changes };\n\t}\n\n\tprivate setFontInfo(changes: FontTokensUpdate): void {\n\t\tthis._onDidChangeFontTokens.fire({ changes });\n\t}\n\n\tprivate refreshAllVisibleLineTokens(): void {\n\t\tconst ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));\n\t\tthis.refreshRanges(ranges);\n\t}\n\n\tprivate refreshRanges(ranges: readonly LineRange[]): void {\n\t\tfor (const range of ranges) {\n\t\t\tthis.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);\n\t\t}\n\t}\n\n\tprivate refreshRange(startLineNumber: number, endLineNumber: number): void {\n\t\tif (!this._tokenizer) {\n\t\t\treturn;\n\t\t}\n\n\t\tstartLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));\n\t\tendLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);\n\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tconst { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);\n\t\tconst changedTokens = this.setTokens(builder.finalize());\n\n\t\tif (heuristicTokens) {\n\t\t\t// We overrode tokens with heuristically computed ones.\n\t\t\t// Because old states might get reused (thus stopping invalidation),\n\t\t\t// we have to explicitly request the tokens for the changed ranges again.\n\t\t\tfor (const c of changedTokens.changes) {\n\t\t\t\tthis._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);\n\t\t\t}\n\t\t}\n\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic forceTokenization(lineNumber: number): void {\n\t\tconst builder = new ContiguousMultilineTokensBuilder();\n\t\tthis._tokenizer?.updateTokensUntilLine(builder, lineNumber);\n\t\tthis.setTokens(builder.finalize());\n\t\tthis._defaultBackgroundTokenizer?.checkFinished();\n\t}\n\n\tpublic hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic isCheapToTokenize(lineNumber: number): boolean {\n\t\tif (!this._tokenizer) {\n\t\t\treturn true;\n\t\t}\n\t\treturn this._tokenizer.isCheapToTokenize(lineNumber);\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst lineText = this._textModel.getLineContent(lineNumber);\n\t\tconst result = this._tokens.getTokens(\n\t\t\tthis._textModel.getLanguageId(),\n\t\t\tlineNumber - 1,\n\t\t\tlineText\n\t\t);\n\t\tif (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {\n\t\t\tif (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {\n\t\t\t\tconst backgroundResult = this._debugBackgroundTokens.getTokens(\n\t\t\t\t\tthis._textModel.getLanguageId(),\n\t\t\t\t\tlineNumber - 1,\n\t\t\t\t\tlineText\n\t\t\t\t);\n\t\t\t\tif (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {\n\t\t\t\t\tthis._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn result;\n\t}\n\n\tpublic getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\tif (!this._tokenizer) {\n\t\t\treturn StandardTokenType.Other;\n\t\t}\n\n\t\tconst position = this._textModel.validatePosition(new Position(lineNumber, column));\n\t\tthis.forceTokenization(position.lineNumber);\n\t\treturn this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);\n\t}\n\n\n\tpublic tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tif (!this._tokenizer) {\n\t\t\treturn null;\n\t\t}\n\t\tthis.forceTokenization(lineNumber);\n\t\treturn this._tokenizer.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic get hasTokens(): boolean {\n\t\treturn this._tokens.hasTokens;\n\t}\n}\n"]}
package/esm/vs/editor/common/textModelEditSource.js CHANGED
@@ -71,6 +71,7 @@ export const EditSources = {
71
71
  source: 'inlineCompletionAccept',
72
72
  $nes: data.nes,
73
73
  ...toProperties(data.providerId),
74
+ $$correlationId: data.correlationId,
74
75
  $$requestUuid: data.requestUuid,
75
76
  $$languageId: data.languageId,
76
77
  });
@@ -81,6 +82,7 @@ export const EditSources = {
81
82
  type: data.type,
82
83
  $nes: data.nes,
83
84
  ...toProperties(data.providerId),
85
+ $$correlationId: data.correlationId,
84
86
  $$requestUuid: data.requestUuid,
85
87
  $$languageId: data.languageId,
86
88
  });
package/esm/vs/editor/common/textModelEditSource.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/textModelEditSource.ts","vs/editor/common/textModelEditSource.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,MAAM,aAAa,GAAG,MAAM,CAAC,qBAAqB,CAAC,CAAC;AAEpD,MAAM,OAAO,mBAAmB;IAC/B,YACiB,QAAsC,EACtD,iBAAuC;QADvB,aAAQ,GAAR,QAAQ,CAA8B;IAEnD,CAAC;IAEE,QAAQ;QACd,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAClC,CAAC;IAEM,OAAO;QACb,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,QAAQ;gBACZ,OAAO,QAAQ,CAAC,IAAI,CAAC;YACtB,KAAK,wBAAwB;gBAC5B,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxD,KAAK,SAAS;gBACb,OAAO,QAAQ,CAAC,IAAI,IAAI,SAAS,CAAC;YACnC;gBACC,OAAO,QAAQ,CAAC,MAAM,CAAC;QACzB,CAAC;IACF,CAAC;IAED;;;MAGE;IACK,KAAK,CAAC,KAAa,EAAE,SAAmE,EAAE;QAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC7D,MAAM,SAAS,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAC;YAC3D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC7B,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACpD,OAAO,WAAW,IAAI,KAAK,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACtF,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;CACD;AAMD,8DAA8D;AAC9D,SAAS,gBAAgB,CAAgC,QAAW;IACnE,uFAAuF;IACvF,OAAO,IAAI,mBAAmB,CAAC,QAAe,EAAE,aAAa,CAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG;IAC1B,OAAO,CAAC,IAA8B;QACrC,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;SACN,CAAC,CAAC;IACb,CAAC;IAED,MAAM,EAAE,CAAC,OAA2B,EAAE,OAAe,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAW,CAAC;IAEnJ,cAAc,CAAC,IAQd;QACA,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,iBAAiB;YACzB,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1C,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW;YAC3C,iBAAiB,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO;YAC5C,YAAY,EAAE,IAAI,CAAC,UAAU;YAC7B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,uBAAuB,EAAE,IAAI,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACb,CAAC;IAED,aAAa,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAW,CAAC;IAC5E,SAAS,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAW,CAAC;IAEpE,sBAAsB,CAAC,IAAwF;QAC9G,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,wBAAwB;YAChC,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,aAAa,EAAE,IAAI,CAAC,WAAW;YAC/B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,6BAA6B,CAAC,IAA+G;QAC5I,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,+BAA+B;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,aAAa,EAAE,IAAI,CAAC,WAAW;YAC/B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,mBAAmB,CAAC,IAAsK;QACzL,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,uBAAuB;YAC/B,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1C,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW;YAC3C,iBAAiB,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO;YAC5C,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,cAAc,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAW,CAAC;IAE7E,MAAM,CAAC,IAAsJ;QAC5J,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,cAAc,EAAE,IAAI,CAAC,cAAc;SAC1B,CAAC,CAAC;IACb,CAAC;IAED,QAAQ,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAW,CAAC;IACjE,SAAS,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,WAAW,EAAW,CAAC;IACnE,UAAU,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAW,CAAC;IACrE,OAAO,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAW,CAAC;IAC/D,OAAO,EAAE,CAAC,IAA4C,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,EAAW,CAAC;IAE7I,UAAU,EAAE,CAAC,IAAsE,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,EAAW,CAAC;CAC/L,CAAC;AAEF,SAAS,YAAY,CAAC,OAA+B;IACpD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO;QACN,YAAY,EAAE,OAAO,CAAC,WAAW;QACjC,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;QAC3C,WAAW,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC;AACH,CAAC;AAOD,SAAS,kBAAkB,CAAC,GAAuB;IAClD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,+CAA+C;IAC/C,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAGD,MAAM,OAAO,aAAa;IAClB,MAAM,CAAC,QAAQ,CAAC,IAAY;QAClC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/B,OAAO,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAEM,MAAM,CAAC,SAAS,CACtB,UAA8B,EAC9B,YAAgC,EAChC,UAA8B,EAC9B,YAAgC;QAEhC,IAAI,UAAU,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACtH,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,IAAI,aAAa,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;IAC9E,CAAC;IAED,YACiB,UAAkB,EAClB,YAAoB,EACpB,UAAkB,EAClB,YAAoB;QAHpB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAQ;IACjC,CAAC;CACL;AASD,MAAM,KAAW,gBAAgB,CAQhC;AARD,WAAiB,gBAAgB;IAChC;;MAEE;IACF,SAAgB,KAAK,CAAC,eAAwC;QAC7D,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAHe,sBAAK,QAGpB,CAAA;AACF,CAAC,EARgB,gBAAgB,KAAhB,gBAAgB,QAQhC;AAED,SAAS,cAAc,CAAC,EAAU;IACjC,OAAO,EAAiC,CAAC;AAC1C,CAAC","file":"textModelEditSource.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["\nimport { prefixedUuid } from '../../base/common/uuid.js';\nimport { TextLength } from './core/text/textLength.js';\nimport { ProviderId, VersionedExtensionId } from './languages.js';\n\nconst privateSymbol = Symbol('TextModelEditSource');\n\nexport class TextModelEditSource {\n\tconstructor(\n\t\tpublic readonly metadata: ITextModelEditSourceMetadata,\n\t\t_privateCtorGuard: typeof privateSymbol,\n\t) { }\n\n\tpublic toString(): string {\n\t\treturn `${this.metadata.source}`;\n\t}\n\n\tpublic getType(): string {\n\t\tconst metadata = this.metadata;\n\t\tswitch (metadata.source) {\n\t\t\tcase 'cursor':\n\t\t\t\treturn metadata.kind;\n\t\t\tcase 'inlineCompletionAccept':\n\t\t\t\treturn metadata.source + (metadata.$nes ? ':nes' : '');\n\t\t\tcase 'unknown':\n\t\t\t\treturn metadata.name || 'unknown';\n\t\t\tdefault:\n\t\t\t\treturn metadata.source;\n\t\t}\n\t}\n\n\t/**\n\t * Converts the metadata to a key string.\n\t * Only includes properties/values that have `level` many `$` prefixes or less.\n\t*/\n\tpublic toKey(level: number, filter: { [TKey in ITextModelEditSourceMetadataKeys]?: boolean } = {}): string {\n\t\tconst metadata = this.metadata;\n\t\tconst keys = Object.entries(metadata).filter(([key, value]) => {\n\t\t\tconst filterVal = (filter as Record<string, boolean>)[key];\n\t\t\tif (filterVal !== undefined) {\n\t\t\t\treturn filterVal;\n\t\t\t}\n\n\t\t\tconst prefixCount = (key.match(/\\$/g) || []).length;\n\t\t\treturn prefixCount <= level && value !== undefined && value !== null && value !== '';\n\t\t}).map(([key, value]) => `${key}:${value}`);\n\t\treturn keys.join('-');\n\t}\n}\n\ntype TextModelEditSourceT<T> = TextModelEditSource & {\n\tmetadataT: T;\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction createEditSource<T extends Record<string, any>>(metadata: T): TextModelEditSourceT<T> {\n\t// eslint-disable-next-line local/code-no-any-casts, @typescript-eslint/no-explicit-any\n\treturn new TextModelEditSource(metadata as any, privateSymbol) as any;\n}\n\nexport const EditSources = {\n\tunknown(data: { name?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'unknown',\n\t\t\tname: data.name,\n\t\t} as const);\n\t},\n\n\trename: (oldName: string | undefined, newName: string) => createEditSource({ source: 'rename', $$$oldName: oldName, $$$newName: newName } as const),\n\n\tchatApplyEdits(data: {\n\t\tmodelId: string | undefined;\n\t\tsessionId: string | undefined;\n\t\trequestId: string | undefined;\n\t\tlanguageId: string;\n\t\tmode: string | undefined;\n\t\textensionId: VersionedExtensionId | undefined;\n\t\tcodeBlockSuggestionId: EditSuggestionId | undefined;\n\t}) {\n\t\treturn createEditSource({\n\t\t\tsource: 'Chat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$languageId: data.languageId,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$mode: data.mode,\n\t\t\t$$codeBlockSuggestionId: data.codeBlockSuggestionId,\n\t\t} as const);\n\t},\n\n\tchatUndoEdits: () => createEditSource({ source: 'Chat.undoEdits' } as const),\n\tchatReset: () => createEditSource({ source: 'Chat.reset' } as const),\n\n\tinlineCompletionAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionAccept',\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineCompletionPartialAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; type: 'word' | 'line' }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionPartialAccept',\n\t\t\ttype: data.type,\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineChatApplyEdit(data: { modelId: string | undefined; requestId: string | undefined; sessionId: string | undefined; languageId: string; extensionId: VersionedExtensionId | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineChat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\treloadFromDisk: () => createEditSource({ source: 'reloadFromDisk' } as const),\n\n\tcursor(data: { kind: 'compositionType' | 'compositionEnd' | 'type' | 'paste' | 'cut' | 'executeCommands' | 'executeCommand'; detailedSource?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'cursor',\n\t\t\tkind: data.kind,\n\t\t\tdetailedSource: data.detailedSource,\n\t\t} as const);\n\t},\n\n\tsetValue: () => createEditSource({ source: 'setValue' } as const),\n\teolChange: () => createEditSource({ source: 'eolChange' } as const),\n\tapplyEdits: () => createEditSource({ source: 'applyEdits' } as const),\n\tsnippet: () => createEditSource({ source: 'snippet' } as const),\n\tsuggest: (data: { providerId: ProviderId | undefined }) => createEditSource({ source: 'suggest', ...toProperties(data.providerId) } as const),\n\n\tcodeAction: (data: { kind: string | undefined; providerId: ProviderId | undefined }) => createEditSource({ source: 'codeAction', $kind: data.kind, ...toProperties(data.providerId) } as const)\n};\n\nfunction toProperties(version: ProviderId | undefined) {\n\tif (!version) {\n\t\treturn {};\n\t}\n\treturn {\n\t\t$extensionId: version.extensionId,\n\t\t$extensionVersion: version.extensionVersion,\n\t\t$providerId: version.providerId,\n\t};\n}\n\ntype Values<T> = T[keyof T];\nexport type ITextModelEditSourceMetadata = Values<{ [TKey in keyof typeof EditSources]: ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\ntype ITextModelEditSourceMetadataKeys = Values<{ [TKey in keyof typeof EditSources]: keyof ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\n\n\nfunction avoidPathRedaction(str: string | undefined): string | undefined {\n\tif (str === undefined) {\n\t\treturn undefined;\n\t}\n\t// To avoid false-positive file path redaction.\n\treturn str.replaceAll('/', '|');\n}\n\n\nexport class EditDeltaInfo {\n\tpublic static fromText(text: string): EditDeltaInfo {\n\t\tconst linesAdded = TextLength.ofText(text).lineCount;\n\t\tconst charsAdded = text.length;\n\t\treturn new EditDeltaInfo(linesAdded, 0, charsAdded, 0);\n\t}\n\n\tpublic static tryCreate(\n\t\tlinesAdded: number | undefined,\n\t\tlinesRemoved: number | undefined,\n\t\tcharsAdded: number | undefined,\n\t\tcharsRemoved: number | undefined\n\t): EditDeltaInfo | undefined {\n\t\tif (linesAdded === undefined || linesRemoved === undefined || charsAdded === undefined || charsRemoved === undefined) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn new EditDeltaInfo(linesAdded, linesRemoved, charsAdded, charsRemoved);\n\t}\n\n\tconstructor(\n\t\tpublic readonly linesAdded: number,\n\t\tpublic readonly linesRemoved: number,\n\t\tpublic readonly charsAdded: number,\n\t\tpublic readonly charsRemoved: number\n\t) { }\n}\n\n\n/**\n * This is an opaque serializable type that represents a unique identity for an edit.\n */\nexport interface EditSuggestionId {\n}\n\nexport namespace EditSuggestionId {\n\t/**\n\t * Use AiEditTelemetryServiceImpl to create a new id!\n\t*/\n\texport function newId(genPrefixedUuid?: (ns: string) => string): EditSuggestionId {\n\t\tconst id = genPrefixedUuid ? genPrefixedUuid('sgt') : prefixedUuid('sgt');\n\t\treturn toEditIdentity(id);\n\t}\n}\n\nfunction toEditIdentity(id: string): EditSuggestionId {\n\treturn id as unknown as EditSuggestionId;\n}\n","\nimport { prefixedUuid } from '../../base/common/uuid.js';\nimport { TextLength } from './core/text/textLength.js';\nimport { ProviderId, VersionedExtensionId } from './languages.js';\n\nconst privateSymbol = Symbol('TextModelEditSource');\n\nexport class TextModelEditSource {\n\tconstructor(\n\t\tpublic readonly metadata: ITextModelEditSourceMetadata,\n\t\t_privateCtorGuard: typeof privateSymbol,\n\t) { }\n\n\tpublic toString(): string {\n\t\treturn `${this.metadata.source}`;\n\t}\n\n\tpublic getType(): string {\n\t\tconst metadata = this.metadata;\n\t\tswitch (metadata.source) {\n\t\t\tcase 'cursor':\n\t\t\t\treturn metadata.kind;\n\t\t\tcase 'inlineCompletionAccept':\n\t\t\t\treturn metadata.source + (metadata.$nes ? ':nes' : '');\n\t\t\tcase 'unknown':\n\t\t\t\treturn metadata.name || 'unknown';\n\t\t\tdefault:\n\t\t\t\treturn metadata.source;\n\t\t}\n\t}\n\n\t/**\n\t * Converts the metadata to a key string.\n\t * Only includes properties/values that have `level` many `$` prefixes or less.\n\t*/\n\tpublic toKey(level: number, filter: { [TKey in ITextModelEditSourceMetadataKeys]?: boolean } = {}): string {\n\t\tconst metadata = this.metadata;\n\t\tconst keys = Object.entries(metadata).filter(([key, value]) => {\n\t\t\tconst filterVal = (filter as Record<string, boolean>)[key];\n\t\t\tif (filterVal !== undefined) {\n\t\t\t\treturn filterVal;\n\t\t\t}\n\n\t\t\tconst prefixCount = (key.match(/\\$/g) || []).length;\n\t\t\treturn prefixCount <= level && value !== undefined && value !== null && value !== '';\n\t\t}).map(([key, value]) => `${key}:${value}`);\n\t\treturn keys.join('-');\n\t}\n}\n\ntype TextModelEditSourceT<T> = TextModelEditSource & {\n\tmetadataT: T;\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction createEditSource<T extends Record<string, any>>(metadata: T): TextModelEditSourceT<T> {\n\t// eslint-disable-next-line local/code-no-any-casts, @typescript-eslint/no-explicit-any\n\treturn new TextModelEditSource(metadata as any, privateSymbol) as any;\n}\n\nexport const EditSources = {\n\tunknown(data: { name?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'unknown',\n\t\t\tname: data.name,\n\t\t} as const);\n\t},\n\n\trename: (oldName: string | undefined, newName: string) => createEditSource({ source: 'rename', $$$oldName: oldName, $$$newName: newName } as const),\n\n\tchatApplyEdits(data: {\n\t\tmodelId: string | undefined;\n\t\tsessionId: string | undefined;\n\t\trequestId: string | undefined;\n\t\tlanguageId: string;\n\t\tmode: string | undefined;\n\t\textensionId: VersionedExtensionId | undefined;\n\t\tcodeBlockSuggestionId: EditSuggestionId | undefined;\n\t}) {\n\t\treturn createEditSource({\n\t\t\tsource: 'Chat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$languageId: data.languageId,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$mode: data.mode,\n\t\t\t$$codeBlockSuggestionId: data.codeBlockSuggestionId,\n\t\t} as const);\n\t},\n\n\tchatUndoEdits: () => createEditSource({ source: 'Chat.undoEdits' } as const),\n\tchatReset: () => createEditSource({ source: 'Chat.reset' } as const),\n\n\tinlineCompletionAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionAccept',\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineCompletionPartialAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; type: 'word' | 'line' }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionPartialAccept',\n\t\t\ttype: data.type,\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineChatApplyEdit(data: { modelId: string | undefined; requestId: string | undefined; sessionId: string | undefined; languageId: string; extensionId: VersionedExtensionId | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineChat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\treloadFromDisk: () => createEditSource({ source: 'reloadFromDisk' } as const),\n\n\tcursor(data: { kind: 'compositionType' | 'compositionEnd' | 'type' | 'paste' | 'cut' | 'executeCommands' | 'executeCommand'; detailedSource?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'cursor',\n\t\t\tkind: data.kind,\n\t\t\tdetailedSource: data.detailedSource,\n\t\t} as const);\n\t},\n\n\tsetValue: () => createEditSource({ source: 'setValue' } as const),\n\teolChange: () => createEditSource({ source: 'eolChange' } as const),\n\tapplyEdits: () => createEditSource({ source: 'applyEdits' } as const),\n\tsnippet: () => createEditSource({ source: 'snippet' } as const),\n\tsuggest: (data: { providerId: ProviderId | undefined }) => createEditSource({ source: 'suggest', ...toProperties(data.providerId) } as const),\n\n\tcodeAction: (data: { kind: string | undefined; providerId: ProviderId | undefined }) => createEditSource({ source: 'codeAction', $kind: data.kind, ...toProperties(data.providerId) } as const)\n};\n\nfunction toProperties(version: ProviderId | undefined) {\n\tif (!version) {\n\t\treturn {};\n\t}\n\treturn {\n\t\t$extensionId: version.extensionId,\n\t\t$extensionVersion: version.extensionVersion,\n\t\t$providerId: version.providerId,\n\t};\n}\n\ntype Values<T> = T[keyof T];\nexport type ITextModelEditSourceMetadata = Values<{ [TKey in keyof typeof EditSources]: ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\ntype ITextModelEditSourceMetadataKeys = Values<{ [TKey in keyof typeof EditSources]: keyof ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\n\n\nfunction avoidPathRedaction(str: string | undefined): string | undefined {\n\tif (str === undefined) {\n\t\treturn undefined;\n\t}\n\t// To avoid false-positive file path redaction.\n\treturn str.replaceAll('/', '|');\n}\n\n\nexport class EditDeltaInfo {\n\tpublic static fromText(text: string): EditDeltaInfo {\n\t\tconst linesAdded = TextLength.ofText(text).lineCount;\n\t\tconst charsAdded = text.length;\n\t\treturn new EditDeltaInfo(linesAdded, 0, charsAdded, 0);\n\t}\n\n\tpublic static tryCreate(\n\t\tlinesAdded: number | undefined,\n\t\tlinesRemoved: number | undefined,\n\t\tcharsAdded: number | undefined,\n\t\tcharsRemoved: number | undefined\n\t): EditDeltaInfo | undefined {\n\t\tif (linesAdded === undefined || linesRemoved === undefined || charsAdded === undefined || charsRemoved === undefined) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn new EditDeltaInfo(linesAdded, linesRemoved, charsAdded, charsRemoved);\n\t}\n\n\tconstructor(\n\t\tpublic readonly linesAdded: number,\n\t\tpublic readonly linesRemoved: number,\n\t\tpublic readonly charsAdded: number,\n\t\tpublic readonly charsRemoved: number\n\t) { }\n}\n\n\n/**\n * This is an opaque serializable type that represents a unique identity for an edit.\n */\nexport interface EditSuggestionId {\n}\n\nexport namespace EditSuggestionId {\n\t/**\n\t * Use AiEditTelemetryServiceImpl to create a new id!\n\t*/\n\texport function newId(genPrefixedUuid?: (ns: string) => string): EditSuggestionId {\n\t\tconst id = genPrefixedUuid ? genPrefixedUuid('sgt') : prefixedUuid('sgt');\n\t\treturn toEditIdentity(id);\n\t}\n}\n\nfunction toEditIdentity(id: string): EditSuggestionId {\n\treturn id as unknown as EditSuggestionId;\n}\n"]}
1
+ {"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/textModelEditSource.ts","vs/editor/common/textModelEditSource.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAGvD,MAAM,aAAa,GAAG,MAAM,CAAC,qBAAqB,CAAC,CAAC;AAEpD,MAAM,OAAO,mBAAmB;IAC/B,YACiB,QAAsC,EACtD,iBAAuC;QADvB,aAAQ,GAAR,QAAQ,CAA8B;IAEnD,CAAC;IAEE,QAAQ;QACd,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAClC,CAAC;IAEM,OAAO;QACb,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,QAAQ;gBACZ,OAAO,QAAQ,CAAC,IAAI,CAAC;YACtB,KAAK,wBAAwB;gBAC5B,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxD,KAAK,SAAS;gBACb,OAAO,QAAQ,CAAC,IAAI,IAAI,SAAS,CAAC;YACnC;gBACC,OAAO,QAAQ,CAAC,MAAM,CAAC;QACzB,CAAC;IACF,CAAC;IAED;;;MAGE;IACK,KAAK,CAAC,KAAa,EAAE,SAAmE,EAAE;QAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC7D,MAAM,SAAS,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAC;YAC3D,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC7B,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACpD,OAAO,WAAW,IAAI,KAAK,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACtF,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;CACD;AAMD,8DAA8D;AAC9D,SAAS,gBAAgB,CAAgC,QAAW;IACnE,uFAAuF;IACvF,OAAO,IAAI,mBAAmB,CAAC,QAAe,EAAE,aAAa,CAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG;IAC1B,OAAO,CAAC,IAA8B;QACrC,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;SACN,CAAC,CAAC;IACb,CAAC;IAED,MAAM,EAAE,CAAC,OAA2B,EAAE,OAAe,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAW,CAAC;IAEnJ,cAAc,CAAC,IAQd;QACA,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,iBAAiB;YACzB,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1C,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW;YAC3C,iBAAiB,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO;YAC5C,YAAY,EAAE,IAAI,CAAC,UAAU;YAC7B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,uBAAuB,EAAE,IAAI,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACb,CAAC;IAED,aAAa,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAW,CAAC;IAC5E,SAAS,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAW,CAAC;IAEpE,sBAAsB,CAAC,IAA2H;QACjJ,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,wBAAwB;YAChC,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,eAAe,EAAE,IAAI,CAAC,aAAa;YACnC,aAAa,EAAE,IAAI,CAAC,WAAW;YAC/B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,6BAA6B,CAAC,IAAkJ;QAC/K,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,+BAA+B;YACvC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,GAAG;YACd,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;YAChC,eAAe,EAAE,IAAI,CAAC,aAAa;YACnC,aAAa,EAAE,IAAI,CAAC,WAAW;YAC/B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,mBAAmB,CAAC,IAAsK;QACzL,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,uBAAuB;YAC/B,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1C,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW;YAC3C,iBAAiB,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO;YAC5C,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,WAAW,EAAE,IAAI,CAAC,SAAS;YAC3B,YAAY,EAAE,IAAI,CAAC,UAAU;SACpB,CAAC,CAAC;IACb,CAAC;IAED,cAAc,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAW,CAAC;IAE7E,MAAM,CAAC,IAAsJ;QAC5J,OAAO,gBAAgB,CAAC;YACvB,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,cAAc,EAAE,IAAI,CAAC,cAAc;SAC1B,CAAC,CAAC;IACb,CAAC;IAED,QAAQ,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAW,CAAC;IACjE,SAAS,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,WAAW,EAAW,CAAC;IACnE,UAAU,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAW,CAAC;IACrE,OAAO,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAW,CAAC;IAC/D,OAAO,EAAE,CAAC,IAA4C,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,EAAW,CAAC;IAE7I,UAAU,EAAE,CAAC,IAAsE,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,EAAW,CAAC;CAC/L,CAAC;AAEF,SAAS,YAAY,CAAC,OAA+B;IACpD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO;QACN,YAAY,EAAE,OAAO,CAAC,WAAW;QACjC,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;QAC3C,WAAW,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC;AACH,CAAC;AAOD,SAAS,kBAAkB,CAAC,GAAuB;IAClD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,+CAA+C;IAC/C,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AAGD,MAAM,OAAO,aAAa;IAClB,MAAM,CAAC,QAAQ,CAAC,IAAY;QAClC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/B,OAAO,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IAEM,MAAM,CAAC,SAAS,CACtB,UAA8B,EAC9B,YAAgC,EAChC,UAA8B,EAC9B,YAAgC;QAEhC,IAAI,UAAU,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACtH,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,IAAI,aAAa,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;IAC9E,CAAC;IAED,YACiB,UAAkB,EAClB,YAAoB,EACpB,UAAkB,EAClB,YAAoB;QAHpB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAQ;IACjC,CAAC;CACL;AASD,MAAM,KAAW,gBAAgB,CAQhC;AARD,WAAiB,gBAAgB;IAChC;;MAEE;IACF,SAAgB,KAAK,CAAC,eAAwC;QAC7D,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAHe,sBAAK,QAGpB,CAAA;AACF,CAAC,EARgB,gBAAgB,KAAhB,gBAAgB,QAQhC;AAED,SAAS,cAAc,CAAC,EAAU;IACjC,OAAO,EAAiC,CAAC;AAC1C,CAAC","file":"textModelEditSource.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["\nimport { prefixedUuid } from '../../base/common/uuid.js';\nimport { TextLength } from './core/text/textLength.js';\nimport { ProviderId, VersionedExtensionId } from './languages.js';\n\nconst privateSymbol = Symbol('TextModelEditSource');\n\nexport class TextModelEditSource {\n\tconstructor(\n\t\tpublic readonly metadata: ITextModelEditSourceMetadata,\n\t\t_privateCtorGuard: typeof privateSymbol,\n\t) { }\n\n\tpublic toString(): string {\n\t\treturn `${this.metadata.source}`;\n\t}\n\n\tpublic getType(): string {\n\t\tconst metadata = this.metadata;\n\t\tswitch (metadata.source) {\n\t\t\tcase 'cursor':\n\t\t\t\treturn metadata.kind;\n\t\t\tcase 'inlineCompletionAccept':\n\t\t\t\treturn metadata.source + (metadata.$nes ? ':nes' : '');\n\t\t\tcase 'unknown':\n\t\t\t\treturn metadata.name || 'unknown';\n\t\t\tdefault:\n\t\t\t\treturn metadata.source;\n\t\t}\n\t}\n\n\t/**\n\t * Converts the metadata to a key string.\n\t * Only includes properties/values that have `level` many `$` prefixes or less.\n\t*/\n\tpublic toKey(level: number, filter: { [TKey in ITextModelEditSourceMetadataKeys]?: boolean } = {}): string {\n\t\tconst metadata = this.metadata;\n\t\tconst keys = Object.entries(metadata).filter(([key, value]) => {\n\t\t\tconst filterVal = (filter as Record<string, boolean>)[key];\n\t\t\tif (filterVal !== undefined) {\n\t\t\t\treturn filterVal;\n\t\t\t}\n\n\t\t\tconst prefixCount = (key.match(/\\$/g) || []).length;\n\t\t\treturn prefixCount <= level && value !== undefined && value !== null && value !== '';\n\t\t}).map(([key, value]) => `${key}:${value}`);\n\t\treturn keys.join('-');\n\t}\n}\n\ntype TextModelEditSourceT<T> = TextModelEditSource & {\n\tmetadataT: T;\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction createEditSource<T extends Record<string, any>>(metadata: T): TextModelEditSourceT<T> {\n\t// eslint-disable-next-line local/code-no-any-casts, @typescript-eslint/no-explicit-any\n\treturn new TextModelEditSource(metadata as any, privateSymbol) as any;\n}\n\nexport const EditSources = {\n\tunknown(data: { name?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'unknown',\n\t\t\tname: data.name,\n\t\t} as const);\n\t},\n\n\trename: (oldName: string | undefined, newName: string) => createEditSource({ source: 'rename', $$$oldName: oldName, $$$newName: newName } as const),\n\n\tchatApplyEdits(data: {\n\t\tmodelId: string | undefined;\n\t\tsessionId: string | undefined;\n\t\trequestId: string | undefined;\n\t\tlanguageId: string;\n\t\tmode: string | undefined;\n\t\textensionId: VersionedExtensionId | undefined;\n\t\tcodeBlockSuggestionId: EditSuggestionId | undefined;\n\t}) {\n\t\treturn createEditSource({\n\t\t\tsource: 'Chat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$languageId: data.languageId,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$mode: data.mode,\n\t\t\t$$codeBlockSuggestionId: data.codeBlockSuggestionId,\n\t\t} as const);\n\t},\n\n\tchatUndoEdits: () => createEditSource({ source: 'Chat.undoEdits' } as const),\n\tchatReset: () => createEditSource({ source: 'Chat.reset' } as const),\n\n\tinlineCompletionAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; correlationId: string | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionAccept',\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$correlationId: data.correlationId,\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineCompletionPartialAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; correlationId: string | undefined; type: 'word' | 'line' }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionPartialAccept',\n\t\t\ttype: data.type,\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$correlationId: data.correlationId,\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineChatApplyEdit(data: { modelId: string | undefined; requestId: string | undefined; sessionId: string | undefined; languageId: string; extensionId: VersionedExtensionId | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineChat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\treloadFromDisk: () => createEditSource({ source: 'reloadFromDisk' } as const),\n\n\tcursor(data: { kind: 'compositionType' | 'compositionEnd' | 'type' | 'paste' | 'cut' | 'executeCommands' | 'executeCommand'; detailedSource?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'cursor',\n\t\t\tkind: data.kind,\n\t\t\tdetailedSource: data.detailedSource,\n\t\t} as const);\n\t},\n\n\tsetValue: () => createEditSource({ source: 'setValue' } as const),\n\teolChange: () => createEditSource({ source: 'eolChange' } as const),\n\tapplyEdits: () => createEditSource({ source: 'applyEdits' } as const),\n\tsnippet: () => createEditSource({ source: 'snippet' } as const),\n\tsuggest: (data: { providerId: ProviderId | undefined }) => createEditSource({ source: 'suggest', ...toProperties(data.providerId) } as const),\n\n\tcodeAction: (data: { kind: string | undefined; providerId: ProviderId | undefined }) => createEditSource({ source: 'codeAction', $kind: data.kind, ...toProperties(data.providerId) } as const)\n};\n\nfunction toProperties(version: ProviderId | undefined) {\n\tif (!version) {\n\t\treturn {};\n\t}\n\treturn {\n\t\t$extensionId: version.extensionId,\n\t\t$extensionVersion: version.extensionVersion,\n\t\t$providerId: version.providerId,\n\t};\n}\n\ntype Values<T> = T[keyof T];\nexport type ITextModelEditSourceMetadata = Values<{ [TKey in keyof typeof EditSources]: ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\ntype ITextModelEditSourceMetadataKeys = Values<{ [TKey in keyof typeof EditSources]: keyof ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\n\n\nfunction avoidPathRedaction(str: string | undefined): string | undefined {\n\tif (str === undefined) {\n\t\treturn undefined;\n\t}\n\t// To avoid false-positive file path redaction.\n\treturn str.replaceAll('/', '|');\n}\n\n\nexport class EditDeltaInfo {\n\tpublic static fromText(text: string): EditDeltaInfo {\n\t\tconst linesAdded = TextLength.ofText(text).lineCount;\n\t\tconst charsAdded = text.length;\n\t\treturn new EditDeltaInfo(linesAdded, 0, charsAdded, 0);\n\t}\n\n\tpublic static tryCreate(\n\t\tlinesAdded: number | undefined,\n\t\tlinesRemoved: number | undefined,\n\t\tcharsAdded: number | undefined,\n\t\tcharsRemoved: number | undefined\n\t): EditDeltaInfo | undefined {\n\t\tif (linesAdded === undefined || linesRemoved === undefined || charsAdded === undefined || charsRemoved === undefined) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn new EditDeltaInfo(linesAdded, linesRemoved, charsAdded, charsRemoved);\n\t}\n\n\tconstructor(\n\t\tpublic readonly linesAdded: number,\n\t\tpublic readonly linesRemoved: number,\n\t\tpublic readonly charsAdded: number,\n\t\tpublic readonly charsRemoved: number\n\t) { }\n}\n\n\n/**\n * This is an opaque serializable type that represents a unique identity for an edit.\n */\nexport interface EditSuggestionId {\n}\n\nexport namespace EditSuggestionId {\n\t/**\n\t * Use AiEditTelemetryServiceImpl to create a new id!\n\t*/\n\texport function newId(genPrefixedUuid?: (ns: string) => string): EditSuggestionId {\n\t\tconst id = genPrefixedUuid ? genPrefixedUuid('sgt') : prefixedUuid('sgt');\n\t\treturn toEditIdentity(id);\n\t}\n}\n\nfunction toEditIdentity(id: string): EditSuggestionId {\n\treturn id as unknown as EditSuggestionId;\n}\n","\nimport { prefixedUuid } from '../../base/common/uuid.js';\nimport { TextLength } from './core/text/textLength.js';\nimport { ProviderId, VersionedExtensionId } from './languages.js';\n\nconst privateSymbol = Symbol('TextModelEditSource');\n\nexport class TextModelEditSource {\n\tconstructor(\n\t\tpublic readonly metadata: ITextModelEditSourceMetadata,\n\t\t_privateCtorGuard: typeof privateSymbol,\n\t) { }\n\n\tpublic toString(): string {\n\t\treturn `${this.metadata.source}`;\n\t}\n\n\tpublic getType(): string {\n\t\tconst metadata = this.metadata;\n\t\tswitch (metadata.source) {\n\t\t\tcase 'cursor':\n\t\t\t\treturn metadata.kind;\n\t\t\tcase 'inlineCompletionAccept':\n\t\t\t\treturn metadata.source + (metadata.$nes ? ':nes' : '');\n\t\t\tcase 'unknown':\n\t\t\t\treturn metadata.name || 'unknown';\n\t\t\tdefault:\n\t\t\t\treturn metadata.source;\n\t\t}\n\t}\n\n\t/**\n\t * Converts the metadata to a key string.\n\t * Only includes properties/values that have `level` many `$` prefixes or less.\n\t*/\n\tpublic toKey(level: number, filter: { [TKey in ITextModelEditSourceMetadataKeys]?: boolean } = {}): string {\n\t\tconst metadata = this.metadata;\n\t\tconst keys = Object.entries(metadata).filter(([key, value]) => {\n\t\t\tconst filterVal = (filter as Record<string, boolean>)[key];\n\t\t\tif (filterVal !== undefined) {\n\t\t\t\treturn filterVal;\n\t\t\t}\n\n\t\t\tconst prefixCount = (key.match(/\\$/g) || []).length;\n\t\t\treturn prefixCount <= level && value !== undefined && value !== null && value !== '';\n\t\t}).map(([key, value]) => `${key}:${value}`);\n\t\treturn keys.join('-');\n\t}\n}\n\ntype TextModelEditSourceT<T> = TextModelEditSource & {\n\tmetadataT: T;\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction createEditSource<T extends Record<string, any>>(metadata: T): TextModelEditSourceT<T> {\n\t// eslint-disable-next-line local/code-no-any-casts, @typescript-eslint/no-explicit-any\n\treturn new TextModelEditSource(metadata as any, privateSymbol) as any;\n}\n\nexport const EditSources = {\n\tunknown(data: { name?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'unknown',\n\t\t\tname: data.name,\n\t\t} as const);\n\t},\n\n\trename: (oldName: string | undefined, newName: string) => createEditSource({ source: 'rename', $$$oldName: oldName, $$$newName: newName } as const),\n\n\tchatApplyEdits(data: {\n\t\tmodelId: string | undefined;\n\t\tsessionId: string | undefined;\n\t\trequestId: string | undefined;\n\t\tlanguageId: string;\n\t\tmode: string | undefined;\n\t\textensionId: VersionedExtensionId | undefined;\n\t\tcodeBlockSuggestionId: EditSuggestionId | undefined;\n\t}) {\n\t\treturn createEditSource({\n\t\t\tsource: 'Chat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$languageId: data.languageId,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$mode: data.mode,\n\t\t\t$$codeBlockSuggestionId: data.codeBlockSuggestionId,\n\t\t} as const);\n\t},\n\n\tchatUndoEdits: () => createEditSource({ source: 'Chat.undoEdits' } as const),\n\tchatReset: () => createEditSource({ source: 'Chat.reset' } as const),\n\n\tinlineCompletionAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; correlationId: string | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionAccept',\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$correlationId: data.correlationId,\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineCompletionPartialAccept(data: { nes: boolean; requestUuid: string; languageId: string; providerId?: ProviderId; correlationId: string | undefined; type: 'word' | 'line' }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineCompletionPartialAccept',\n\t\t\ttype: data.type,\n\t\t\t$nes: data.nes,\n\t\t\t...toProperties(data.providerId),\n\t\t\t$$correlationId: data.correlationId,\n\t\t\t$$requestUuid: data.requestUuid,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\tinlineChatApplyEdit(data: { modelId: string | undefined; requestId: string | undefined; sessionId: string | undefined; languageId: string; extensionId: VersionedExtensionId | undefined }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'inlineChat.applyEdits',\n\t\t\t$modelId: avoidPathRedaction(data.modelId),\n\t\t\t$extensionId: data.extensionId?.extensionId,\n\t\t\t$extensionVersion: data.extensionId?.version,\n\t\t\t$$sessionId: data.sessionId,\n\t\t\t$$requestId: data.requestId,\n\t\t\t$$languageId: data.languageId,\n\t\t} as const);\n\t},\n\n\treloadFromDisk: () => createEditSource({ source: 'reloadFromDisk' } as const),\n\n\tcursor(data: { kind: 'compositionType' | 'compositionEnd' | 'type' | 'paste' | 'cut' | 'executeCommands' | 'executeCommand'; detailedSource?: string | null }) {\n\t\treturn createEditSource({\n\t\t\tsource: 'cursor',\n\t\t\tkind: data.kind,\n\t\t\tdetailedSource: data.detailedSource,\n\t\t} as const);\n\t},\n\n\tsetValue: () => createEditSource({ source: 'setValue' } as const),\n\teolChange: () => createEditSource({ source: 'eolChange' } as const),\n\tapplyEdits: () => createEditSource({ source: 'applyEdits' } as const),\n\tsnippet: () => createEditSource({ source: 'snippet' } as const),\n\tsuggest: (data: { providerId: ProviderId | undefined }) => createEditSource({ source: 'suggest', ...toProperties(data.providerId) } as const),\n\n\tcodeAction: (data: { kind: string | undefined; providerId: ProviderId | undefined }) => createEditSource({ source: 'codeAction', $kind: data.kind, ...toProperties(data.providerId) } as const)\n};\n\nfunction toProperties(version: ProviderId | undefined) {\n\tif (!version) {\n\t\treturn {};\n\t}\n\treturn {\n\t\t$extensionId: version.extensionId,\n\t\t$extensionVersion: version.extensionVersion,\n\t\t$providerId: version.providerId,\n\t};\n}\n\ntype Values<T> = T[keyof T];\nexport type ITextModelEditSourceMetadata = Values<{ [TKey in keyof typeof EditSources]: ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\ntype ITextModelEditSourceMetadataKeys = Values<{ [TKey in keyof typeof EditSources]: keyof ReturnType<typeof EditSources[TKey]>['metadataT'] }>;\n\n\nfunction avoidPathRedaction(str: string | undefined): string | undefined {\n\tif (str === undefined) {\n\t\treturn undefined;\n\t}\n\t// To avoid false-positive file path redaction.\n\treturn str.replaceAll('/', '|');\n}\n\n\nexport class EditDeltaInfo {\n\tpublic static fromText(text: string): EditDeltaInfo {\n\t\tconst linesAdded = TextLength.ofText(text).lineCount;\n\t\tconst charsAdded = text.length;\n\t\treturn new EditDeltaInfo(linesAdded, 0, charsAdded, 0);\n\t}\n\n\tpublic static tryCreate(\n\t\tlinesAdded: number | undefined,\n\t\tlinesRemoved: number | undefined,\n\t\tcharsAdded: number | undefined,\n\t\tcharsRemoved: number | undefined\n\t): EditDeltaInfo | undefined {\n\t\tif (linesAdded === undefined || linesRemoved === undefined || charsAdded === undefined || charsRemoved === undefined) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn new EditDeltaInfo(linesAdded, linesRemoved, charsAdded, charsRemoved);\n\t}\n\n\tconstructor(\n\t\tpublic readonly linesAdded: number,\n\t\tpublic readonly linesRemoved: number,\n\t\tpublic readonly charsAdded: number,\n\t\tpublic readonly charsRemoved: number\n\t) { }\n}\n\n\n/**\n * This is an opaque serializable type that represents a unique identity for an edit.\n */\nexport interface EditSuggestionId {\n}\n\nexport namespace EditSuggestionId {\n\t/**\n\t * Use AiEditTelemetryServiceImpl to create a new id!\n\t*/\n\texport function newId(genPrefixedUuid?: (ns: string) => string): EditSuggestionId {\n\t\tconst id = genPrefixedUuid ? genPrefixedUuid('sgt') : prefixedUuid('sgt');\n\t\treturn toEditIdentity(id);\n\t}\n}\n\nfunction toEditIdentity(id: string): EditSuggestionId {\n\treturn id as unknown as EditSuggestionId;\n}\n"]}