@codingame/monaco-vscode-api 9.0.3 → 10.0.0

package/vscode/src/vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase.js

@@ -50,16 +50,11 @@ class TreeNode {
         this.right = null;
     }
 }
-var NodeColor;
-( (function(NodeColor) {
-    NodeColor[NodeColor["Black"] = 0] = "Black";
-    NodeColor[NodeColor["Red"] = 1] = "Red";
-})(NodeColor || (NodeColor = {})));
-const SENTINEL = ( new TreeNode(null, NodeColor.Black));
+const SENTINEL = ( new TreeNode(null, 0 ));
 SENTINEL.parent = SENTINEL;
 SENTINEL.left = SENTINEL;
 SENTINEL.right = SENTINEL;
-SENTINEL.color = NodeColor.Black;
+SENTINEL.color = 0 ;
 function leftest(node) {
     while (node.left !== SENTINEL) {
         node = node.left;
@@ -146,13 +141,13 @@ function rbDelete(tree, z) {
     }
     if (y === tree.root) {
         tree.root = x;
-        x.color = NodeColor.Black;
+        x.color = 0 ;
         z.detach();
         resetSentinel();
         tree.root.parent = SENTINEL;
         return;
     }
-    const yWasRed = (y.color === NodeColor.Red);
+    const yWasRed = ((y.color === 1) );
     if (y === y.parent.left) {
         y.parent.left = x;
     }
@@ -214,72 +209,72 @@ function rbDelete(tree, z) {
         return;
     }
     let w;
-    while (x !== tree.root && x.color === NodeColor.Black) {
+    while (x !== tree.root && x.color === 0 ) {
         if (x === x.parent.left) {
             w = x.parent.right;
-            if (w.color === NodeColor.Red) {
-                w.color = NodeColor.Black;
-                x.parent.color = NodeColor.Red;
+            if (w.color === 1 ) {
+                w.color = 0 ;
+                x.parent.color = 1 ;
                 leftRotate(tree, x.parent);
                 w = x.parent.right;
             }
-            if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
-                w.color = NodeColor.Red;
+            if (w.left.color === 0  && w.right.color === 0 ) {
+                w.color = 1 ;
                 x = x.parent;
             }
             else {
-                if (w.right.color === NodeColor.Black) {
-                    w.left.color = NodeColor.Black;
-                    w.color = NodeColor.Red;
+                if (w.right.color === 0 ) {
+                    w.left.color = 0 ;
+                    w.color = 1 ;
                     rightRotate(tree, w);
                     w = x.parent.right;
                 }
                 w.color = x.parent.color;
-                x.parent.color = NodeColor.Black;
-                w.right.color = NodeColor.Black;
+                x.parent.color = 0 ;
+                w.right.color = 0 ;
                 leftRotate(tree, x.parent);
                 x = tree.root;
             }
         }
         else {
             w = x.parent.left;
-            if (w.color === NodeColor.Red) {
-                w.color = NodeColor.Black;
-                x.parent.color = NodeColor.Red;
+            if (w.color === 1 ) {
+                w.color = 0 ;
+                x.parent.color = 1 ;
                 rightRotate(tree, x.parent);
                 w = x.parent.left;
             }
-            if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
-                w.color = NodeColor.Red;
+            if (w.left.color === 0  && w.right.color === 0 ) {
+                w.color = 1 ;
                 x = x.parent;
             }
             else {
-                if (w.left.color === NodeColor.Black) {
-                    w.right.color = NodeColor.Black;
-                    w.color = NodeColor.Red;
+                if (w.left.color === 0 ) {
+                    w.right.color = 0 ;
+                    w.color = 1 ;
                     leftRotate(tree, w);
                     w = x.parent.left;
                 }
                 w.color = x.parent.color;
-                x.parent.color = NodeColor.Black;
-                w.left.color = NodeColor.Black;
+                x.parent.color = 0 ;
+                w.left.color = 0 ;
                 rightRotate(tree, x.parent);
                 x = tree.root;
             }
         }
     }
-    x.color = NodeColor.Black;
+    x.color = 0 ;
     resetSentinel();
 }
 function fixInsert(tree, x) {
     recomputeTreeMetadata(tree, x);
-    while (x !== tree.root && x.parent.color === NodeColor.Red) {
+    while (x !== tree.root && x.parent.color === 1 ) {
         if (x.parent === x.parent.parent.left) {
             const y = x.parent.parent.right;
-            if (y.color === NodeColor.Red) {
-                x.parent.color = NodeColor.Black;
-                y.color = NodeColor.Black;
-                x.parent.parent.color = NodeColor.Red;
+            if (y.color === 1 ) {
+                x.parent.color = 0 ;
+                y.color = 0 ;
+                x.parent.parent.color = 1 ;
                 x = x.parent.parent;
             }
             else {
@@ -287,17 +282,17 @@ function fixInsert(tree, x) {
                     x = x.parent;
                     leftRotate(tree, x);
                 }
-                x.parent.color = NodeColor.Black;
-                x.parent.parent.color = NodeColor.Red;
+                x.parent.color = 0 ;
+                x.parent.parent.color = 1 ;
                 rightRotate(tree, x.parent.parent);
             }
         }
         else {
             const y = x.parent.parent.left;
-            if (y.color === NodeColor.Red) {
-                x.parent.color = NodeColor.Black;
-                y.color = NodeColor.Black;
-                x.parent.parent.color = NodeColor.Red;
+            if (y.color === 1 ) {
+                x.parent.color = 0 ;
+                y.color = 0 ;
+                x.parent.parent.color = 1 ;
                 x = x.parent.parent;
             }
             else {
@@ -305,13 +300,13 @@ function fixInsert(tree, x) {
                     x = x.parent;
                     rightRotate(tree, x);
                 }
-                x.parent.color = NodeColor.Black;
-                x.parent.parent.color = NodeColor.Red;
+                x.parent.color = 0 ;
+                x.parent.parent.color = 1 ;
                 leftRotate(tree, x.parent.parent);
             }
         }
     }
-    tree.root.color = NodeColor.Black;
+    tree.root.color = 0 ;
 }
 function updateTreeMetadata(tree, x, delta, lineFeedCntDelta) {
     while (x !== tree.root && x !== SENTINEL) {
@@ -347,4 +342,4 @@ function recomputeTreeMetadata(tree, x) {
         x = x.parent;
     }
 }
-export { NodeColor, SENTINEL, TreeNode, fixInsert, leftRotate, leftest, rbDelete, recomputeTreeMetadata, rightRotate, righttest, updateTreeMetadata };
+export { SENTINEL, TreeNode, fixInsert, leftRotate, leftest, rbDelete, recomputeTreeMetadata, rightRotate, righttest, updateTreeMetadata };

package/vscode/src/vs/editor/common/model/textModel.js

@@ -6,7 +6,6 @@ import { Emitter } from '../../../base/common/event.js';
 import { Disposable, combinedDisposable, MutableDisposable } from '../../../base/common/lifecycle.js';
 import { listenStream } from '../../../base/common/stream.js';
 import { singleLetterHash, UNUSUAL_LINE_TERMINATORS, isHighSurrogate, htmlAttributeEncodeValue } from '../../../base/common/strings.js';
-import { Constants } from '../../../base/common/uint.js';
 import { URI } from '../../../base/common/uri.js';
 import { countEOL } from '../core/eolCounter.js';
 import { normalizeIndentation } from '../core/indentation.js';
@@ -16,7 +15,7 @@ import { Selection } from '../core/selection.js';
 import { EDITOR_MODEL_DEFAULTS } from '../core/textModelDefaults.js';
 import { ILanguageService } from '../languages/language.js';
 import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.js';
-import { DefaultEndOfLine, TextModelResolvedOptions, EndOfLinePreference, EndOfLineSequence, ValidAnnotatedEditOperation, TrackedRangeStickiness, isITextSnapshot, OverviewRulerLane, GlyphMarginLane } from '../model.js';
+import { TextModelResolvedOptions, ValidAnnotatedEditOperation, isITextSnapshot, OverviewRulerLane, GlyphMarginLane } from '../model.js';
 import { BracketPairsTextModelPart } from './bracketPairsTextModelPart/bracketPairsImpl.js';
 import { ColorizedBracketPairsDecorationProvider } from './bracketPairsTextModelPart/colorizedBracketPairsDecorationProvider.js';
 import { EditStack } from './editStack.js';
@@ -118,11 +117,6 @@ class TextModelSnapshot {
     }
 }
 const invalidFunc = () => { throw ( new Error(`Invalid change accessor`)); };
-var StringOffsetValidationType;
-( (function(StringOffsetValidationType) {
-    StringOffsetValidationType[StringOffsetValidationType["Relaxed"] = 0] = "Relaxed";
-    StringOffsetValidationType[StringOffsetValidationType["SurrogatePairs"] = 1] = "SurrogatePairs";
-})(StringOffsetValidationType || (StringOffsetValidationType = {})));
 let TextModel = class TextModel extends Disposable {
     static { TextModel_1 = this; }
     static { this._MODEL_SYNC_LIMIT = 50 * 1024 * 1024; }
@@ -135,7 +129,7 @@ let TextModel = class TextModel extends Disposable {
         indentSize: EDITOR_MODEL_DEFAULTS.indentSize,
         insertSpaces: EDITOR_MODEL_DEFAULTS.insertSpaces,
         detectIndentation: false,
-        defaultEOL: DefaultEndOfLine.LF,
+        defaultEOL: 1 ,
         trimAutoWhitespace: EDITOR_MODEL_DEFAULTS.trimAutoWhitespace,
         largeFileOptimizations: EDITOR_MODEL_DEFAULTS.largeFileOptimizations,
         bracketPairColorizationOptions: EDITOR_MODEL_DEFAULTS.bracketPairColorizationOptions,
@@ -211,7 +205,7 @@ let TextModel = class TextModel extends Disposable {
         this._decorationProvider = this._register(( new ColorizedBracketPairsDecorationProvider(this)));
         this._tokenizationTextModelPart = this.instantiationService.createInstance(TokenizationTextModelPart, this, this._bracketPairs, languageId, this._attachedViews);
         const bufferLineCount = this._buffer.getLineCount();
-        const bufferTextLength = this._buffer.getValueLengthInRange(( new Range(1, 1, bufferLineCount, this._buffer.getLineLength(bufferLineCount) + 1)), EndOfLinePreference.TextDefined);
+        const bufferTextLength = this._buffer.getValueLengthInRange(( new Range(1, 1, bufferLineCount, this._buffer.getLineLength(bufferLineCount) + 1)), 0 );
         if (creationOptions.largeFileOptimizations) {
             this._isTooLargeForTokenization = ((bufferTextLength > TextModel_1.LARGE_FILE_SIZE_THRESHOLD)
                 || (bufferLineCount > TextModel_1.LARGE_FILE_LINE_COUNT_THRESHOLD));
@@ -333,7 +327,7 @@ let TextModel = class TextModel extends Disposable {
     }
     setEOL(eol) {
         this._assertNotDisposed();
-        const newEOL = (eol === EndOfLineSequence.CRLF ? '\r\n' : '\n');
+        const newEOL = (eol === 1  ? '\r\n' : '\n');
         if (this._buffer.getEOL() === newEOL) {
             return;
         }
@@ -483,7 +477,7 @@ let TextModel = class TextModel extends Disposable {
         return this._buffer.mightContainUnusualLineTerminators();
     }
     removeUnusualLineTerminators(selections = null) {
-        const matches = this.findMatches(UNUSUAL_LINE_TERMINATORS.source, false, true, false, null, false, Constants.MAX_SAFE_SMALL_INTEGER);
+        const matches = this.findMatches(UNUSUAL_LINE_TERMINATORS.source, false, true, false, null, false, 1073741824 );
         this._buffer.resetMightContainUnusualLineTerminators();
         this.pushEditOperations(selections, ( matches.map(m => ({ range: m.range, text: null }))), () => null);
     }
@@ -500,7 +494,7 @@ let TextModel = class TextModel extends Disposable {
     }
     getOffsetAt(rawPosition) {
         this._assertNotDisposed();
-        const position = this._validatePosition(rawPosition.lineNumber, rawPosition.column, StringOffsetValidationType.Relaxed);
+        const position = this._validatePosition(rawPosition.lineNumber, rawPosition.column, 0 );
         return this._buffer.getOffsetAt(position.lineNumber, position.column);
     }
     getPositionAt(rawOffset) {
@@ -545,15 +539,15 @@ let TextModel = class TextModel extends Disposable {
         }
         return fullModelValue;
     }
-    getValueInRange(rawRange, eol = EndOfLinePreference.TextDefined) {
+    getValueInRange(rawRange, eol = 0 ) {
         this._assertNotDisposed();
         return this._buffer.getValueInRange(this.validateRange(rawRange), eol);
     }
-    getValueLengthInRange(rawRange, eol = EndOfLinePreference.TextDefined) {
+    getValueLengthInRange(rawRange, eol = 0 ) {
         this._assertNotDisposed();
         return this._buffer.getValueLengthInRange(this.validateRange(rawRange), eol);
     }
-    getCharacterCountInRange(rawRange, eol = EndOfLinePreference.TextDefined) {
+    getCharacterCountInRange(rawRange, eol = 0 ) {
         this._assertNotDisposed();
         return this._buffer.getCharacterCountInRange(this.validateRange(rawRange), eol);
     }
@@ -588,9 +582,7 @@ let TextModel = class TextModel extends Disposable {
     }
     getEndOfLineSequence() {
         this._assertNotDisposed();
-        return (this._buffer.getEOL() === '\n'
-            ? EndOfLineSequence.LF
-            : EndOfLineSequence.CRLF);
+        return (this._buffer.getEOL() === '\n' ? 0  : 1) ;
     }
     getLineMinColumn(lineNumber) {
         this._assertNotDisposed();
@@ -699,7 +691,7 @@ let TextModel = class TextModel extends Disposable {
         if (column > maxColumn) {
             return false;
         }
-        if (validationType === StringOffsetValidationType.SurrogatePairs) {
+        if (validationType === 1 ) {
             const charCodeBefore = this._buffer.getLineCharCode(lineNumber, column - 2);
             if (isHighSurrogate(charCodeBefore)) {
                 return false;
@@ -724,7 +716,7 @@ let TextModel = class TextModel extends Disposable {
         if (column >= maxColumn) {
             return ( new Position(lineNumber, maxColumn));
         }
-        if (validationType === StringOffsetValidationType.SurrogatePairs) {
+        if (validationType === 1 ) {
             const charCodeBefore = this._buffer.getLineCharCode(lineNumber, column - 2);
             if (isHighSurrogate(charCodeBefore)) {
                 return ( new Position(lineNumber, column - 1));
@@ -733,7 +725,7 @@ let TextModel = class TextModel extends Disposable {
         return ( new Position(lineNumber, column));
     }
     validatePosition(position) {
-        const validationType = StringOffsetValidationType.SurrogatePairs;
+        const validationType = 1 ;
         this._assertNotDisposed();
         if (position instanceof Position) {
             if (this._isValidPosition(position.lineNumber, position.column, validationType)) {
@@ -747,13 +739,13 @@ let TextModel = class TextModel extends Disposable {
         const startColumn = range.startColumn;
         const endLineNumber = range.endLineNumber;
         const endColumn = range.endColumn;
-        if (!this._isValidPosition(startLineNumber, startColumn, StringOffsetValidationType.Relaxed)) {
+        if (!this._isValidPosition(startLineNumber, startColumn, 0 )) {
             return false;
         }
-        if (!this._isValidPosition(endLineNumber, endColumn, StringOffsetValidationType.Relaxed)) {
+        if (!this._isValidPosition(endLineNumber, endColumn, 0 )) {
             return false;
         }
-        if (validationType === StringOffsetValidationType.SurrogatePairs) {
+        if (validationType === 1 ) {
             const charCodeBeforeStart = (startColumn > 1 ? this._buffer.getLineCharCode(startLineNumber, startColumn - 2) : 0);
             const charCodeBeforeEnd = (endColumn > 1 && endColumn <= this._buffer.getLineLength(endLineNumber) ? this._buffer.getLineCharCode(endLineNumber, endColumn - 2) : 0);
             const startInsideSurrogatePair = isHighSurrogate(charCodeBeforeStart);
@@ -766,20 +758,20 @@ let TextModel = class TextModel extends Disposable {
         return true;
     }
     validateRange(_range) {
-        const validationType = StringOffsetValidationType.SurrogatePairs;
+        const validationType = 1 ;
         this._assertNotDisposed();
         if ((_range instanceof Range) && !(_range instanceof Selection)) {
             if (this._isValidRange(_range, validationType)) {
                 return _range;
             }
         }
-        const start = this._validatePosition(_range.startLineNumber, _range.startColumn, StringOffsetValidationType.Relaxed);
-        const end = this._validatePosition(_range.endLineNumber, _range.endColumn, StringOffsetValidationType.Relaxed);
+        const start = this._validatePosition(_range.startLineNumber, _range.startColumn, 0 );
+        const end = this._validatePosition(_range.endLineNumber, _range.endColumn, 0 );
         const startLineNumber = start.lineNumber;
         const startColumn = start.column;
         const endLineNumber = end.lineNumber;
         const endColumn = end.column;
-        if (validationType === StringOffsetValidationType.SurrogatePairs) {
+        {
             const charCodeBeforeStart = (startColumn > 1 ? this._buffer.getLineCharCode(startLineNumber, startColumn - 2) : 0);
             const charCodeBeforeEnd = (endColumn > 1 && endColumn <= this._buffer.getLineLength(endLineNumber) ? this._buffer.getLineCharCode(endLineNumber, endColumn - 2) : 0);
             const startInsideSurrogatePair = isHighSurrogate(charCodeBeforeStart);
@@ -798,7 +790,6 @@ let TextModel = class TextModel extends Disposable {
             }
             return ( new Range(startLineNumber, startColumn, endLineNumber, endColumn + 1));
         }
-        return ( new Range(startLineNumber, startColumn, endLineNumber, endColumn));
     }
     modifyPosition(rawPosition, offset) {
         this._assertNotDisposed();
@@ -897,7 +888,7 @@ let TextModel = class TextModel extends Disposable {
         this._commandManager.popStackElement();
     }
     pushEOL(eol) {
-        const currentEOL = (this.getEOL() === '\n' ? EndOfLineSequence.LF : EndOfLineSequence.CRLF);
+        const currentEOL = ((this.getEOL() === '\n' ? 0  : 1) );
         if (currentEOL === eol) {
             return;
         }
@@ -1715,6 +1706,7 @@ class ModelDecorationInjectedTextOptions {
     }
     constructor(options) {
         this.content = options.content || '';
+        this.tokens = options.tokens ?? null;
         this.inlineClassName = options.inlineClassName || null;
         this.inlineClassNameAffectsLetterSpacing = options.inlineClassNameAffectsLetterSpacing || false;
         this.attachedData = options.attachedData || null;
@@ -1734,7 +1726,7 @@ class ModelDecorationOptions {
         this.blockDoesNotCollapse = options.blockDoesNotCollapse ?? null;
         this.blockIsAfterEnd = options.blockIsAfterEnd ?? null;
         this.blockPadding = options.blockPadding ?? null;
-        this.stickiness = options.stickiness || TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges;
+        this.stickiness = options.stickiness || 0 ;
         this.zIndex = options.zIndex || 0;
         this.className = options.className ? cleanClassName(options.className) : null;
         this.shouldFillLineOnLineBreak = options.shouldFillLineOnLineBreak ?? null;
@@ -1765,10 +1757,10 @@ class ModelDecorationOptions {
 }
 ModelDecorationOptions.EMPTY = ModelDecorationOptions.register({ description: 'empty' });
 const TRACKED_RANGE_OPTIONS = [
-    ModelDecorationOptions.register({ description: 'tracked-range-always-grows-when-typing-at-edges', stickiness: TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges }),
-    ModelDecorationOptions.register({ description: 'tracked-range-never-grows-when-typing-at-edges', stickiness: TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges }),
-    ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-before', stickiness: TrackedRangeStickiness.GrowsOnlyWhenTypingBefore }),
-    ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-after', stickiness: TrackedRangeStickiness.GrowsOnlyWhenTypingAfter }),
+    ModelDecorationOptions.register({ description: 'tracked-range-always-grows-when-typing-at-edges', stickiness: 0  }),
+    ModelDecorationOptions.register({ description: 'tracked-range-never-grows-when-typing-at-edges', stickiness: 1  }),
+    ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-before', stickiness: 2  }),
+    ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-after', stickiness: 3  }),
 ];
 function _normalizeOptions(options) {
     if (options instanceof ModelDecorationOptions) {

package/vscode/src/vs/editor/common/model/textModelSearch.js

@@ -1,9 +1,8 @@
-import { CharCode } from '../../../base/common/charCode.js';
 import { createRegExp, getNextCodePoint } from '../../../base/common/strings.js';
-import { getMapForWordSeparators, WordCharacterClass } from '../core/wordCharacterClassifier.js';
+import { getMapForWordSeparators } from '../core/wordCharacterClassifier.js';
 import { Position } from '../core/position.js';
 import { Range } from '../core/range.js';
-import { SearchData, EndOfLinePreference, FindMatch } from '../model.js';
+import { SearchData, FindMatch } from '../model.js';
 const LIMIT_FIND_COUNT = 999;
 class SearchParams {
     constructor(searchString, isRegex, matchCase, wordSeparators) {
@@ -56,16 +55,16 @@ function isMultilineRegexSource(searchString) {
     }
     for (let i = 0, len = searchString.length; i < len; i++) {
         const chCode = searchString.charCodeAt(i);
-        if (chCode === CharCode.LineFeed) {
+        if (chCode === 10 ) {
             return true;
         }
-        if (chCode === CharCode.Backslash) {
+        if (chCode === 92 ) {
             i++;
             if (i >= len) {
                 break;
             }
             const nextChCode = searchString.charCodeAt(i);
-            if (nextChCode === CharCode.n || nextChCode === CharCode.r || nextChCode === CharCode.W) {
+            if (nextChCode === 110  || nextChCode === 114  || nextChCode === 87 ) {
                 return true;
             }
         }
@@ -87,7 +86,7 @@ class LineFeedCounter {
         const lineFeedsOffsets = [];
         let lineFeedsOffsetsLen = 0;
         for (let i = 0, textLen = text.length; i < textLen; i++) {
-            if (text.charCodeAt(i) === CharCode.LineFeed) {
+            if (text.charCodeAt(i) === 10 ) {
                 lineFeedsOffsets[lineFeedsOffsetsLen++] = i;
             }
         }
@@ -162,7 +161,7 @@ class TextModelSearch {
     }
     static _doFindMatchesMultiline(model, searchRange, searcher, captureMatches, limitResultCount) {
         const deltaOffset = model.getOffsetAt(searchRange.getStartPosition());
-        const text = model.getValueInRange(searchRange, EndOfLinePreference.LF);
+        const text = model.getValueInRange(searchRange, 1 );
         const lfCounter = (model.getEOL() === '\r\n' ? ( new LineFeedCounter(text)) : null);
         const result = [];
         let counter = 0;
@@ -256,7 +255,7 @@ class TextModelSearch {
             searchTextStart.column,
             lineCount,
             model.getLineMaxColumn(lineCount)
-        )), EndOfLinePreference.LF);
+        )), 1 );
         const lfCounter = (model.getEOL() === '\r\n' ? ( new LineFeedCounter(text)) : null);
         searcher.reset(searchStart.column - 1);
         const m = searcher.next(text);
@@ -349,15 +348,15 @@ function leftIsWordBounday(wordSeparators, text, textLength, matchStartIndex, ma
         return true;
     }
     const charBefore = text.charCodeAt(matchStartIndex - 1);
-    if (wordSeparators.get(charBefore) !== WordCharacterClass.Regular) {
+    if (wordSeparators.get(charBefore) !== 0 ) {
         return true;
     }
-    if (charBefore === CharCode.CarriageReturn || charBefore === CharCode.LineFeed) {
+    if (charBefore === 13  || charBefore === 10 ) {
         return true;
     }
     if (matchLength > 0) {
         const firstCharInMatch = text.charCodeAt(matchStartIndex);
-        if (wordSeparators.get(firstCharInMatch) !== WordCharacterClass.Regular) {
+        if (wordSeparators.get(firstCharInMatch) !== 0 ) {
             return true;
         }
     }
@@ -368,15 +367,15 @@ function rightIsWordBounday(wordSeparators, text, textLength, matchStartIndex, m
         return true;
     }
     const charAfter = text.charCodeAt(matchStartIndex + matchLength);
-    if (wordSeparators.get(charAfter) !== WordCharacterClass.Regular) {
+    if (wordSeparators.get(charAfter) !== 0 ) {
         return true;
     }
-    if (charAfter === CharCode.CarriageReturn || charAfter === CharCode.LineFeed) {
+    if (charAfter === 13  || charAfter === 10 ) {
         return true;
     }
     if (matchLength > 0) {
         const lastCharInMatch = text.charCodeAt(matchStartIndex + matchLength - 1);
-        if (wordSeparators.get(lastCharInMatch) !== WordCharacterClass.Regular) {
+        if (wordSeparators.get(lastCharInMatch) !== 0 ) {
             return true;
         }
     }

package/vscode/src/vs/editor/common/model/textModelText.js

@@ -8,6 +8,9 @@ class TextModelText extends AbstractText {
     getValueOfRange(range) {
         return this._textModel.getValueInRange(range);
     }
+    getLineLength(lineNumber) {
+        return this._textModel.getLineLength(lineNumber);
+    }
     get length() {
         const lastLineNumber = this._textModel.getLineCount();
         const lastLineLen = this._textModel.getLineLength(lastLineNumber);

package/vscode/src/vs/editor/common/model/textModelTokens.js

@@ -5,15 +5,10 @@ import { StopWatch } from '../../../base/common/stopwatch.js';
 import { countEOL } from '../core/eolCounter.js';
 import { LineRange } from '../core/lineRange.js';
 import { OffsetRange } from '../core/offsetRange.js';
-import { StandardTokenType } from '../encodedTokenAttributes.js';
 import { nullTokenizeEncoded } from '../languages/nullTokenize.js';
 import { FixedArray } from './fixedArray.js';
 import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
 import { LineTokens } from '../tokens/lineTokens.js';
-var Constants;
-( (function(Constants) {
-    Constants[Constants["CHEAP_TOKENIZATION_LENGTH_LIMIT"] = 2048] = "CHEAP_TOKENIZATION_LENGTH_LIMIT";
-})(Constants || (Constants = {})));
 class TokenizerWithStateStore {
     constructor(lineCount, tokenizationSupport) {
         this.tokenizationSupport = tokenizationSupport;
@@ -49,7 +44,7 @@ class TokenizerWithStateStoreAndTextModel extends TokenizerWithStateStore {
     getTokenTypeIfInsertingCharacter(position, character) {
         const lineStartState = this.getStartState(position.lineNumber);
         if (!lineStartState) {
-            return StandardTokenType.Other;
+            return 0 ;
         }
         const languageId = this._textModel.getLanguageId();
         const lineContent = this._textModel.getLineContent(position.lineNumber);
@@ -59,25 +54,32 @@ class TokenizerWithStateStoreAndTextModel extends TokenizerWithStateStore {
         const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState);
         const lineTokens = ( new LineTokens(r.tokens, text, this._languageIdCodec));
         if (lineTokens.getCount() === 0) {
-            return StandardTokenType.Other;
+            return 0 ;
         }
         const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
         return lineTokens.getStandardTokenType(tokenIndex);
     }
-    tokenizeLineWithEdit(position, length, newText) {
-        const lineNumber = position.lineNumber;
-        const column = position.column;
+    tokenizeLineWithEdit(lineNumber, edit) {
         const lineStartState = this.getStartState(lineNumber);
         if (!lineStartState) {
-            return null;
+            return { mainLineTokens: null, additionalLines: null };
         }
         const curLineContent = this._textModel.getLineContent(lineNumber);
-        const newLineContent = curLineContent.substring(0, column - 1)
-            + newText + curLineContent.substring(column - 1 + length);
+        const newLineContent = edit.lineEdit.apply(curLineContent);
         const languageId = this._textModel.getLanguageIdAtPosition(lineNumber, 0);
         const result = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, newLineContent, true, lineStartState);
-        const lineTokens = ( new LineTokens(result.tokens, newLineContent, this._languageIdCodec));
-        return lineTokens;
+        let additionalLines = null;
+        if (edit.additionalLines) {
+            additionalLines = [];
+            let state = result.endState;
+            for (const line of edit.additionalLines) {
+                const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, line, true, state);
+                additionalLines.push(( new LineTokens(r.tokens, line, this._languageIdCodec)));
+                state = r.endState;
+            }
+        }
+        const mainLineTokens = ( new LineTokens(result.tokens, newLineContent, this._languageIdCodec));
+        return { mainLineTokens, additionalLines };
     }
     hasAccurateTokensForLine(lineNumber) {
         const firstInvalidLineNumber = this.store.getFirstInvalidEndStateLineNumberOrMax();
@@ -89,7 +91,7 @@ class TokenizerWithStateStoreAndTextModel extends TokenizerWithStateStore {
             return true;
         }
         if (lineNumber === firstInvalidLineNumber
-            && this._textModel.getLineLength(lineNumber) < Constants.CHEAP_TOKENIZATION_LENGTH_LIMIT) {
+            && this._textModel.getLineLength(lineNumber) < 2048 ) {
             return true;
         }
         return false;

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.js

@@ -1,5 +1,4 @@
 import { __decorate, __param } from '../../../../../../external/tslib/tslib.es6.js';
-import { CharCode } from '../../../base/common/charCode.js';
 import { BugIndicatingError, onUnexpectedError } from '../../../base/common/errors.js';
 import { Emitter, Event } from '../../../base/common/event.js';
 import { DisposableStore, MutableDisposable, DisposableMap } from '../../../base/common/lifecycle.js';
@@ -7,7 +6,6 @@ import { countEOL } from '../core/eolCounter.js';
 import { LineRange } from '../core/lineRange.js';
 import { Position } from '../core/position.js';
 import { getWordAtText } from '../core/wordHelper.js';
-import { StandardTokenType } from '../encodedTokenAttributes.js';
 import { TreeSitterTokenizationRegistry, TokenizationRegistry } from '../languages.js';
 import { ILanguageService } from '../languages/language.js';
 import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.js';
@@ -16,7 +14,6 @@ import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, De
 import { AbstractTokens, AttachedViewHandler } from './tokens.js';
 import { TreeSitterTokens } from './treeSitterTokens.js';
 import { ITreeSitterParserService } from '../services/treeSitterParserService.js';
-import { BackgroundTokenizationState } from '../tokenizationTextModelPart.js';
 import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
 import { ContiguousTokensStore } from '../tokens/contiguousTokensStore.js';
 import { SparseTokensStore } from '../tokens/sparseTokensStore.js';
@@ -109,7 +106,7 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
         else if (!e.isEolChange) {
             for (const c of e.changes) {
                 const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
-                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null);
+                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : 0 );
             }
         }
         this._tokens.handleDidChangeContent(e);
@@ -161,8 +158,8 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
     getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
         return this._tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
     }
-    tokenizeLineWithEdit(position, length, newText) {
-        return this._tokens.tokenizeLineWithEdit(position, length, newText);
+    tokenizeLineWithEdit(lineNumber, edit) {
+        return this._tokens.tokenizeLineWithEdit(lineNumber, edit);
     }
     setSemanticTokens(tokens, isComplete) {
         this._semanticTokens.set(tokens, isComplete);
@@ -358,10 +355,10 @@ class GrammarTokens extends AbstractTokens {
                     this.setTokens(tokens);
                 },
                 backgroundTokenizationFinished: () => {
-                    if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
+                    if (this._backgroundTokenizationState === 2 ) {
                         return;
                     }
-                    const newState = BackgroundTokenizationState.Completed;
+                    const newState = 2 ;
                     this._backgroundTokenizationState = newState;
                     this._onDidChangeBackgroundTokenizationState.fire();
                 },
@@ -491,19 +488,18 @@ class GrammarTokens extends AbstractTokens {
     }
     getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
         if (!this._tokenizer) {
-            return StandardTokenType.Other;
+            return 0 ;
         }
         const position = this._textModel.validatePosition(( new Position(lineNumber, column)));
         this.forceTokenization(position.lineNumber);
         return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
     }
-    tokenizeLineWithEdit(position, length, newText) {
+    tokenizeLineWithEdit(lineNumber, edit) {
         if (!this._tokenizer) {
-            return null;
+            return { mainLineTokens: null, additionalLines: null };
         }
-        const validatedPosition = this._textModel.validatePosition(position);
-        this.forceTokenization(validatedPosition.lineNumber);
-        return this._tokenizer.tokenizeLineWithEdit(validatedPosition, length, newText);
+        this.forceTokenization(lineNumber);
+        return this._tokenizer.tokenizeLineWithEdit(lineNumber, edit);
     }
     get hasTokens() {
         return this._tokens.hasTokens;