@codingame/monaco-vscode-api 16.1.1 → 17.1.0

package/vscode/src/vs/editor/common/model/textModel.js

@@ -29,9 +29,10 @@ import { PieceTreeTextBufferBuilder } from './pieceTreeTextBuffer/pieceTreeTextB
 import { SearchParams, TextModelSearch } from './textModelSearch.js';
 import { TokenizationTextModelPart } from './tokenizationTextModelPart.js';
 import { AttachedViews } from './tokens.js';
-import { InternalModelContentChangeEvent, ModelRawContentChangedEvent, ModelRawFlush, ModelRawEOLChanged, LineInjectedText, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted, ModelInjectedTextChangedEvent } from '../textModelEvents.js';
+import { InternalModelContentChangeEvent, ModelRawContentChangedEvent, ModelRawFlush, ModelRawEOLChanged, LineInjectedText, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted, ModelInjectedTextChangedEvent, ModelLineHeightChanged, ModelLineHeightChangedEvent } from '../textModelEvents.js';
 import { IInstantiationService } from '../../../platform/instantiation/common/instantiation.js';
 import { IUndoRedoService } from '../../../platform/undoRedo/common/undoRedo.service.js';
+import { SetWithKey } from '../../../base/common/collections.js';
 
 var TextModel_1;
 function createTextBufferFactory(text) {
@@ -178,7 +179,7 @@ let TextModel = class TextModel extends Disposable {
         this._onWillDispose = this._register(( new Emitter()));
         this.onWillDispose = this._onWillDispose.event;
         this._onDidChangeDecorations = this._register(( new DidChangeDecorationsEmitter(
-            affectedInjectedTextLines => this.handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines)
+            (affectedInjectedTextLines, affectedLineHeights) => this.handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines, affectedLineHeights)
         )));
         this.onDidChangeDecorations = this._onDidChangeDecorations.event;
         this._onDidChangeOptions = this._register(( new Emitter()));
@@ -186,6 +187,8 @@ let TextModel = class TextModel extends Disposable {
         this._onDidChangeAttached = this._register(( new Emitter()));
         this.onDidChangeAttached = this._onDidChangeAttached.event;
         this._onDidChangeInjectedText = this._register(( new Emitter()));
+        this._onDidChangeLineHeight = this._register(( new Emitter()));
+        this.onDidChangeLineHeight = this._onDidChangeLineHeight.event;
         this._eventEmitter = this._register(( new DidChangeContentEmitter()));
         this._languageSelectionListener = this._register(( new MutableDisposable()));
         this._deltaDecorationCallCnt = 0;
@@ -268,6 +271,7 @@ let TextModel = class TextModel extends Disposable {
             || this._onDidChangeOptions.hasListeners()
             || this._onDidChangeAttached.hasListeners()
             || this._onDidChangeInjectedText.hasListeners()
+            || this._onDidChangeLineHeight.hasListeners()
             || this._eventEmitter.hasListeners());
     }
     _assertNotDisposed() {
@@ -1158,17 +1162,26 @@ let TextModel = class TextModel extends Disposable {
     canRedo() {
         return this._undoRedoService.canRedo(this.uri);
     }
-    handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines) {
-        if (affectedInjectedTextLines === null || affectedInjectedTextLines.size === 0) {
-            return;
+    handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines, affectedLineHeights) {
+        if (affectedInjectedTextLines && affectedInjectedTextLines.size > 0) {
+            const affectedLines = Array.from(affectedInjectedTextLines);
+            const lineChangeEvents = ( affectedLines.map(lineNumber => ( new ModelRawLineChanged(
+                lineNumber,
+                this.getLineContent(lineNumber),
+                this._getInjectedTextInLine(lineNumber)
+            ))));
+            this._onDidChangeInjectedText.fire(( new ModelInjectedTextChangedEvent(lineChangeEvents)));
+        }
+        if (affectedLineHeights && affectedLineHeights.size > 0) {
+            const affectedLines = Array.from(affectedLineHeights);
+            const lineHeightChangeEvent = ( affectedLines.map(specialLineHeightChange => ( new ModelLineHeightChanged(
+                specialLineHeightChange.ownerId,
+                specialLineHeightChange.decorationId,
+                specialLineHeightChange.lineNumber,
+                specialLineHeightChange.lineHeight
+            ))));
+            this._onDidChangeLineHeight.fire(( new ModelLineHeightChangedEvent(lineHeightChangeEvent)));
         }
-        const affectedLines = Array.from(affectedInjectedTextLines);
-        const lineChangeEvents = ( affectedLines.map(lineNumber => ( new ModelRawLineChanged(
-            lineNumber,
-            this.getLineContent(lineNumber),
-            this._getInjectedTextInLine(lineNumber)
-        ))));
-        this._onDidChangeInjectedText.fire(( new ModelInjectedTextChangedEvent(lineChangeEvents)));
     }
     changeDecorations(callback, ownerId = 0) {
         this._assertNotDisposed();
@@ -1186,10 +1199,10 @@ let TextModel = class TextModel extends Disposable {
                 return this._deltaDecorationsImpl(ownerId, [], [{ range: range, options: options }])[0];
             },
             changeDecoration: (id, newRange) => {
-                this._changeDecorationImpl(id, newRange);
+                this._changeDecorationImpl(ownerId, id, newRange);
             },
             changeDecorationOptions: (id, options) => {
-                this._changeDecorationOptionsImpl(id, _normalizeOptions(options));
+                this._changeDecorationOptionsImpl(ownerId, id, _normalizeOptions(options));
             },
             removeDecoration: (id) => {
                 this._deltaDecorationsImpl(ownerId, [id], []);
@@ -1315,6 +1328,9 @@ let TextModel = class TextModel extends Disposable {
     getInjectedTextDecorations(ownerId = 0) {
         return this._decorationsTree.getAllInjectedText(this, ownerId);
     }
+    getCustomLineHeightsDecorations(ownerId = 0) {
+        return this._decorationsTree.getAllCustomLineHeights(this, ownerId);
+    }
     _getInjectedTextInLine(lineNumber) {
         const startOffset = this._buffer.getOffsetAt(lineNumber, 1);
         const endOffset = startOffset + this._buffer.getLineLength(lineNumber);
@@ -1337,7 +1353,7 @@ let TextModel = class TextModel extends Disposable {
     getRangeAt(start, end) {
         return this._buffer.getRangeAt(start, end - start);
     }
-    _changeDecorationImpl(decorationId, _range) {
+    _changeDecorationImpl(ownerId, decorationId, _range) {
         const node = this._decorations[decorationId];
         if (!node) {
             return;
@@ -1350,6 +1366,10 @@ let TextModel = class TextModel extends Disposable {
             const oldRange = this.getDecorationRange(decorationId);
             this._onDidChangeDecorations.recordLineAffectedByInjectedText(oldRange.startLineNumber);
         }
+        if (node.options.lineHeight !== null) {
+            const oldRange = this.getDecorationRange(decorationId);
+            this._onDidChangeDecorations.recordLineAffectedByLineHeightChange(ownerId, decorationId, oldRange.startLineNumber, null);
+        }
         const range = this._validateRangeRelaxedNoAllocations(_range);
         const startOffset = this._buffer.getOffsetAt(range.startLineNumber, range.startColumn);
         const endOffset = this._buffer.getOffsetAt(range.endLineNumber, range.endColumn);
@@ -1363,8 +1383,11 @@ let TextModel = class TextModel extends Disposable {
         if (node.options.before) {
             this._onDidChangeDecorations.recordLineAffectedByInjectedText(range.startLineNumber);
         }
+        if (node.options.lineHeight !== null) {
+            this._onDidChangeDecorations.recordLineAffectedByLineHeightChange(ownerId, decorationId, range.startLineNumber, node.options.lineHeight);
+        }
     }
-    _changeDecorationOptionsImpl(decorationId, options) {
+    _changeDecorationOptionsImpl(ownerId, decorationId, options) {
         const node = this._decorations[decorationId];
         if (!node) {
             return;
@@ -1381,6 +1404,10 @@ let TextModel = class TextModel extends Disposable {
             const nodeRange = this._decorationsTree.getNodeRange(this, node);
             this._onDidChangeDecorations.recordLineAffectedByInjectedText(nodeRange.startLineNumber);
         }
+        if (node.options.lineHeight !== null || options.lineHeight !== null) {
+            const nodeRange = this._decorationsTree.getNodeRange(this, node);
+            this._onDidChangeDecorations.recordLineAffectedByLineHeightChange(ownerId, decorationId, nodeRange.startLineNumber, options.lineHeight);
+        }
         const movedInOverviewRuler = nodeWasInOverviewRuler !== nodeIsInOverviewRuler;
         const changedWhetherInjectedText = isOptionsInjectedText(options) !== isNodeInjectedText(node);
         if (movedInOverviewRuler || changedWhetherInjectedText) {
@@ -1404,8 +1431,10 @@ let TextModel = class TextModel extends Disposable {
             while (oldDecorationIndex < oldDecorationsLen || newDecorationIndex < newDecorationsLen) {
                 let node = null;
                 if (oldDecorationIndex < oldDecorationsLen) {
+                    let decorationId;
                     do {
-                        node = this._decorations[oldDecorationsIds[oldDecorationIndex++]];
+                        decorationId = oldDecorationsIds[oldDecorationIndex++];
+                        node = this._decorations[decorationId];
                     } while (!node && oldDecorationIndex < oldDecorationsLen);
                     if (node) {
                         if (node.options.after) {
@@ -1416,6 +1445,10 @@ let TextModel = class TextModel extends Disposable {
                             const nodeRange = this._decorationsTree.getNodeRange(this, node);
                             this._onDidChangeDecorations.recordLineAffectedByInjectedText(nodeRange.startLineNumber);
                         }
+                        if (node.options.lineHeight !== null) {
+                            const nodeRange = this._decorationsTree.getNodeRange(this, node);
+                            this._onDidChangeDecorations.recordLineAffectedByLineHeightChange(ownerId, decorationId, nodeRange.startLineNumber, null);
+                        }
                         this._decorationsTree.delete(node);
                         if (!suppressEvents) {
                             this._onDidChangeDecorations.checkAffectedAndFire(node.options);
@@ -1443,6 +1476,9 @@ let TextModel = class TextModel extends Disposable {
                     if (node.options.before) {
                         this._onDidChangeDecorations.recordLineAffectedByInjectedText(range.startLineNumber);
                     }
+                    if (node.options.lineHeight !== null) {
+                        this._onDidChangeDecorations.recordLineAffectedByLineHeightChange(ownerId, node.id, range.startLineNumber, node.options.lineHeight);
+                    }
                     if (!suppressEvents) {
                         this._onDidChangeDecorations.checkAffectedAndFire(options);
                     }
@@ -1563,6 +1599,11 @@ class DecorationsTrees {
         const result = this._injectedTextDecorationsTree.search(filterOwnerId, false, versionId, false);
         return this._ensureNodesHaveRanges(host, result).filter((i) => i.options.showIfCollapsed || !i.range.isEmpty());
     }
+    getAllCustomLineHeights(host, filterOwnerId) {
+        const versionId = host.getVersionId();
+        const result = this._search(filterOwnerId, false, false, versionId, false);
+        return this._ensureNodesHaveRanges(host, result).filter((i) => typeof i.options.lineHeight === 'number');
+    }
     getAll(host, filterOwnerId, filterOutValidation, overviewRulerOnly, onlyMarginDecorations) {
         const versionId = host.getVersionId();
         const result = this._search(filterOwnerId, filterOutValidation, overviewRulerOnly, versionId, onlyMarginDecorations);
@@ -1751,6 +1792,7 @@ class ModelDecorationOptions {
         this.glyphMarginHoverMessage = options.glyphMarginHoverMessage || null;
         this.lineNumberHoverMessage = options.lineNumberHoverMessage || null;
         this.isWholeLine = options.isWholeLine || false;
+        this.lineHeight = options.lineHeight ?? null;
         this.showIfCollapsed = options.showIfCollapsed || false;
         this.collapseOnReplaceEdit = options.collapseOnReplaceEdit || false;
         this.overviewRuler = options.overviewRuler ? ( new ModelDecorationOverviewRulerOptions(options.overviewRuler)) : null;
@@ -1785,6 +1827,17 @@ function _normalizeOptions(options) {
     }
     return ModelDecorationOptions.createDynamic(options);
 }
+class LineHeightChangingDecoration {
+    static toKey(obj) {
+        return `${obj.ownerId};${obj.decorationId};${obj.lineNumber}`;
+    }
+    constructor(ownerId, decorationId, lineNumber, lineHeight) {
+        this.ownerId = ownerId;
+        this.decorationId = decorationId;
+        this.lineNumber = lineNumber;
+        this.lineHeight = lineHeight;
+    }
+}
 class DidChangeDecorationsEmitter extends Disposable {
     constructor(handleBeforeFire) {
         super();
@@ -1792,6 +1845,7 @@ class DidChangeDecorationsEmitter extends Disposable {
         this._actual = this._register(( new Emitter()));
         this.event = this._actual.event;
         this._affectedInjectedTextLines = null;
+        this._affectedLineHeights = null;
         this._deferredCnt = 0;
         this._shouldFireDeferred = false;
         this._affectsMinimap = false;
@@ -1813,6 +1867,8 @@ class DidChangeDecorationsEmitter extends Disposable {
             }
             this._affectedInjectedTextLines?.clear();
             this._affectedInjectedTextLines = null;
+            this._affectedLineHeights?.clear();
+            this._affectedLineHeights = null;
         }
     }
     recordLineAffectedByInjectedText(lineNumber) {
@@ -1821,6 +1877,12 @@ class DidChangeDecorationsEmitter extends Disposable {
         }
         this._affectedInjectedTextLines.add(lineNumber);
     }
+    recordLineAffectedByLineHeightChange(ownerId, decorationId, lineNumber, lineHeight) {
+        if (!this._affectedLineHeights) {
+            this._affectedLineHeights = ( new SetWithKey([], LineHeightChangingDecoration.toKey));
+        }
+        this._affectedLineHeights.add(( new LineHeightChangingDecoration(ownerId, decorationId, lineNumber, lineHeight)));
+    }
     checkAffectedAndFire(options) {
         this._affectsMinimap ||= !!options.minimap?.position;
         this._affectsOverviewRuler ||= !!options.overviewRuler?.color;
@@ -1843,7 +1905,7 @@ class DidChangeDecorationsEmitter extends Disposable {
         }
     }
     doFire() {
-        this.handleBeforeFire(this._affectedInjectedTextLines);
+        this.handleBeforeFire(this._affectedInjectedTextLines, this._affectedLineHeights);
         const event = {
             affectsMinimap: this._affectsMinimap,
             affectsOverviewRuler: this._affectsOverviewRuler,

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.js

@@ -157,7 +157,7 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
         return this._tokens.tokenizeLinesAt(lineNumber, lines);
     }
     setSemanticTokens(tokens, isComplete) {
-        this._semanticTokens.set(tokens, isComplete);
+        this._semanticTokens.set(tokens, isComplete, this._textModel);
         this._emitModelTokensChangedEvent({
             semanticTokensApplied: tokens !== null,
             ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],

package/vscode/src/vs/editor/common/model.d.ts

@@ -13,7 +13,7 @@ import { IWordAtPosition } from "./core/wordHelper.js";
 import { FormattingOptions } from "./languages.js";
 import { ILanguageSelection } from "./languages/language.js";
 import { IBracketPairsTextModelPart } from "./textModelBracketPairs.js";
-import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelInjectedTextChangedEvent } from "./textModelEvents.js";
+import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelInjectedTextChangedEvent, ModelLineHeightChangedEvent } from "./textModelEvents.js";
 import { IGuidesTextModelPart } from "./textModelGuides.js";
 import { ITokenizationTextModelPart } from "./tokenizationTextModelPart.js";
 import { UndoRedoGroup } from "../../platform/undoRedo/common/undoRedo.js";
@@ -85,6 +85,7 @@ export interface IModelDecorationOptions {
     minimap?: IModelDecorationMinimapOptions | null;
     glyphMarginClassName?: string | null;
     glyphMargin?: IModelDecorationGlyphMarginOptions | null;
+    lineHeight?: number | null;
     linesDecorationsClassName?: string | null;
     linesDecorationsTooltip?: string | null;
     lineNumberClassName?: string | null;

package/vscode/src/vs/editor/common/standalone/standaloneEnums.d.ts

@@ -273,6 +273,11 @@ export declare enum InlayHintKind {
     Type = 1,
     Parameter = 2
 }
+export declare enum InlineCompletionEndOfLifeReasonKind {
+    Accepted = 0,
+    Rejected = 1,
+    Ignored = 2
+}
 export declare enum InlineCompletionTriggerKind {
     Automatic = 0,
     Explicit = 1

package/vscode/src/vs/editor/common/standalone/standaloneEnums.js

@@ -294,6 +294,12 @@ var InlayHintKind;
     InlayHintKind[InlayHintKind["Type"] = 1] = "Type";
     InlayHintKind[InlayHintKind["Parameter"] = 2] = "Parameter";
 })(InlayHintKind || (InlayHintKind = {}));
+var InlineCompletionEndOfLifeReasonKind;
+(function (InlineCompletionEndOfLifeReasonKind) {
+    InlineCompletionEndOfLifeReasonKind[InlineCompletionEndOfLifeReasonKind["Accepted"] = 0] = "Accepted";
+    InlineCompletionEndOfLifeReasonKind[InlineCompletionEndOfLifeReasonKind["Rejected"] = 1] = "Rejected";
+    InlineCompletionEndOfLifeReasonKind[InlineCompletionEndOfLifeReasonKind["Ignored"] = 2] = "Ignored";
+})(InlineCompletionEndOfLifeReasonKind || (InlineCompletionEndOfLifeReasonKind = {}));
 var InlineCompletionTriggerKind;
 (function (InlineCompletionTriggerKind) {
     InlineCompletionTriggerKind[InlineCompletionTriggerKind["Automatic"] = 0] = "Automatic";
@@ -624,4 +630,4 @@ var WrappingIndent;
     WrappingIndent[WrappingIndent["DeepIndent"] = 3] = "DeepIndent";
 })(WrappingIndent || (WrappingIndent = {}));
 
-export { AccessibilitySupport, CodeActionTriggerType, CompletionItemInsertTextRule, CompletionItemKind, CompletionItemTag, CompletionTriggerKind, ContentWidgetPositionPreference, CursorChangeReason, DefaultEndOfLine, DocumentHighlightKind, EditorAutoIndentStrategy, EditorOption, EndOfLinePreference, EndOfLineSequence, GlyphMarginLane, HoverVerbosityAction, IndentAction, InjectedTextCursorStops, InlayHintKind, InlineCompletionTriggerKind, InlineEditTriggerKind, KeyCode, MarkerSeverity, MarkerTag, MinimapPosition, MinimapSectionHeaderStyle, MouseTargetType, NewSymbolNameTag, NewSymbolNameTriggerKind, OverlayWidgetPositionPreference, OverviewRulerLane, PartialAcceptTriggerKind, PositionAffinity, RenderLineNumbersType, RenderMinimap, ScrollType, ScrollbarVisibility, SelectionDirection, ShowLightbulbIconMode, SignatureHelpTriggerKind, SymbolKind, SymbolTag, TextEditorCursorBlinkingStyle, TextEditorCursorStyle, TrackedRangeStickiness, WrappingIndent };
+export { AccessibilitySupport, CodeActionTriggerType, CompletionItemInsertTextRule, CompletionItemKind, CompletionItemTag, CompletionTriggerKind, ContentWidgetPositionPreference, CursorChangeReason, DefaultEndOfLine, DocumentHighlightKind, EditorAutoIndentStrategy, EditorOption, EndOfLinePreference, EndOfLineSequence, GlyphMarginLane, HoverVerbosityAction, IndentAction, InjectedTextCursorStops, InlayHintKind, InlineCompletionEndOfLifeReasonKind, InlineCompletionTriggerKind, InlineEditTriggerKind, KeyCode, MarkerSeverity, MarkerTag, MinimapPosition, MinimapSectionHeaderStyle, MouseTargetType, NewSymbolNameTag, NewSymbolNameTriggerKind, OverlayWidgetPositionPreference, OverviewRulerLane, PartialAcceptTriggerKind, PositionAffinity, RenderLineNumbersType, RenderMinimap, ScrollType, ScrollbarVisibility, SelectionDirection, ShowLightbulbIconMode, SignatureHelpTriggerKind, SymbolKind, SymbolTag, TextEditorCursorBlinkingStyle, TextEditorCursorStyle, TrackedRangeStickiness, WrappingIndent };

package/vscode/src/vs/editor/common/textModelEvents.d.ts

@@ -70,6 +70,13 @@ export declare class ModelRawLineChanged {
     readonly injectedText: LineInjectedText[] | null;
     constructor(lineNumber: number, detail: string, injectedText: LineInjectedText[] | null);
 }
+export declare class ModelLineHeightChanged {
+    readonly ownerId: number;
+    readonly decorationId: string;
+    readonly lineNumber: number;
+    readonly lineHeight: number | null;
+    constructor(ownerId: number, decorationId: string, lineNumber: number, lineHeight: number | null);
+}
 export declare class ModelRawLinesDeleted {
     readonly changeType = RawContentChangedType.LinesDeleted;
     readonly fromLineNumber: number;
@@ -102,6 +109,10 @@ export declare class ModelInjectedTextChangedEvent {
     readonly changes: ModelRawLineChanged[];
     constructor(changes: ModelRawLineChanged[]);
 }
+export declare class ModelLineHeightChangedEvent {
+    readonly changes: ModelLineHeightChanged[];
+    constructor(changes: ModelLineHeightChanged[]);
+}
 export declare class InternalModelContentChangeEvent {
     readonly rawContentChangedEvent: ModelRawContentChangedEvent;
     readonly contentChangedEvent: IModelContentChangedEvent;

package/vscode/src/vs/editor/common/textModelEvents.js

@@ -86,6 +86,14 @@ class ModelRawLineChanged {
         this.injectedText = injectedText;
     }
 }
+class ModelLineHeightChanged {
+    constructor(ownerId, decorationId, lineNumber, lineHeight) {
+        this.ownerId = ownerId;
+        this.decorationId = decorationId;
+        this.lineNumber = lineNumber;
+        this.lineHeight = lineHeight;
+    }
+}
 class ModelRawLinesDeleted {
     constructor(fromLineNumber, toLineNumber) {
         this.changeType = RawContentChangedType.LinesDeleted;
@@ -137,6 +145,11 @@ class ModelInjectedTextChangedEvent {
         this.changes = changes;
     }
 }
+class ModelLineHeightChangedEvent {
+    constructor(changes) {
+        this.changes = changes;
+    }
+}
 class InternalModelContentChangeEvent {
     constructor(rawContentChangedEvent, contentChangedEvent) {
         this.rawContentChangedEvent = rawContentChangedEvent;
@@ -167,4 +180,4 @@ class InternalModelContentChangeEvent {
     }
 }
 
-export { InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted, RawContentChangedType };
+export { InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelLineHeightChanged, ModelLineHeightChangedEvent, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted, RawContentChangedType };

package/vscode/src/vs/editor/common/tokens/common.d.ts

@@ -0,0 +1,7 @@
+export declare class RateLimiter {
+    readonly timesPerSecond: number;
+    private _lastRun;
+    private readonly _minimumTimeBetweenRuns;
+    constructor(timesPerSecond?: number);
+    runIfNotLimited(callback: () => void): void;
+}

package/vscode/src/vs/editor/common/tokens/common.js

@@ -0,0 +1,18 @@
+
+
+class RateLimiter {
+    constructor(timesPerSecond = 5) {
+        this.timesPerSecond = timesPerSecond;
+        this._lastRun = 0;
+        this._minimumTimeBetweenRuns = 1000 / timesPerSecond;
+    }
+    runIfNotLimited(callback) {
+        const now = Date.now();
+        if (now - this._lastRun >= this._minimumTimeBetweenRuns) {
+            this._lastRun = now;
+            callback();
+        }
+    }
+}
+
+export { RateLimiter };

package/vscode/src/vs/editor/common/tokens/contiguousTokensEditing.d.ts

@@ -6,4 +6,4 @@ export declare class ContiguousTokensEditing {
     static append(lineTokens: Uint32Array | ArrayBuffer | null, _otherTokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer | null;
     static insert(lineTokens: Uint32Array | ArrayBuffer | null, chIndex: number, textLength: number): Uint32Array | ArrayBuffer | null;
 }
-export declare function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array;
+export declare function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array<ArrayBuffer>;

package/vscode/src/vs/editor/common/tokens/lineTokens.d.ts

@@ -36,6 +36,7 @@ export declare class LineTokens implements IViewLineTokens {
     readonly languageIdCodec: ILanguageIdCodec;
     static defaultTokenMetadata: number;
     constructor(tokens: Uint32Array, text: string, decoder: ILanguageIdCodec);
+    getTextLength(): number;
     equals(other: IViewLineTokens): boolean;
     slicedEquals(other: LineTokens, sliceFromTokenIndex: number, sliceTokenCount: number): boolean;
     getLineContent(): string;

package/vscode/src/vs/editor/common/tokens/lineTokens.js

@@ -66,6 +66,9 @@ class LineTokens {
         this._text = text;
         this.languageIdCodec = decoder;
     }
+    getTextLength() {
+        return this._text.length;
+    }
     equals(other) {
         if (other instanceof LineTokens) {
             return this.slicedEquals(other, 0, this._tokensCount);

package/vscode/src/vs/editor/common/tokens/sparseMultilineTokens.d.ts

@@ -1,4 +1,5 @@
 import { IRange, Range } from "../core/range.js";
+import { ITextModel } from "../model.js";
 export declare class SparseMultilineTokens {
     static create(startLineNumber: number, tokens: Uint32Array): SparseMultilineTokens;
     private _startLineNumber;
@@ -21,6 +22,7 @@ export declare class SparseMultilineTokens {
     acceptEdit(range: IRange, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void;
     private _acceptDeleteRange;
     private _acceptInsertText;
+    reportIfInvalid(model: ITextModel): void;
 }
 export declare class SparseLineTokens {
     private readonly _tokens;

package/vscode/src/vs/editor/common/tokens/sparseMultilineTokens.js

@@ -3,6 +3,7 @@ import { CharCode } from '../../../base/common/charCode.js';
 import { Position } from '../core/position.js';
 import { Range } from '../core/range.js';
 import { countEOL } from '../core/eolCounter.js';
+import { RateLimiter } from './common.js';
 
 class SparseMultilineTokens {
     static create(startLineNumber, tokens) {
@@ -111,6 +112,9 @@ class SparseMultilineTokens {
         }
         this._tokens.acceptInsertText(lineIndex, position.column - 1, eolCount, firstLineLength, lastLineLength, firstCharCode);
     }
+    reportIfInvalid(model) {
+        this._tokens.reportIfInvalid(model, this._startLineNumber);
+    }
 }
 class SparseMultilineTokensStorage {
     constructor(tokens) {
@@ -385,6 +389,22 @@ class SparseMultilineTokensStorage {
             tokens[offset + 2] = tokenEndCharacter;
         }
     }
+    static { this._rateLimiter = ( new RateLimiter(10 / 60)); }
+    reportIfInvalid(model, startLineNumber) {
+        for (let i = 0; i < this._tokenCount; i++) {
+            const lineNumber = this._getDeltaLine(i) + startLineNumber;
+            if (lineNumber > model.getLineCount()) {
+                SparseMultilineTokensStorage._rateLimiter.runIfNotLimited(() => {
+                    console.error('Invalid Semantic Tokens Data From Extension: lineNumber > model.getLineCount()');
+                });
+            }
+            if (this._getEndCharacter(i) > model.getLineLength(lineNumber)) {
+                SparseMultilineTokensStorage._rateLimiter.runIfNotLimited(() => {
+                    console.error('Invalid Semantic Tokens Data From Extension: end character > model.getLineLength(lineNumber)');
+                });
+            }
+        }
+    }
 }
 class SparseLineTokens {
     constructor(tokens) {

package/vscode/src/vs/editor/common/tokens/sparseTokensStore.d.ts

@@ -2,6 +2,7 @@ import { IRange, Range } from "../core/range.js";
 import { LineTokens } from "./lineTokens.js";
 import { SparseMultilineTokens } from "./sparseMultilineTokens.js";
 import { ILanguageIdCodec } from "../languages.js";
+import { ITextModel } from "../model.js";
 export declare class SparseTokensStore {
     private _pieces;
     private _isComplete;
@@ -9,7 +10,7 @@ export declare class SparseTokensStore {
     constructor(languageIdCodec: ILanguageIdCodec);
     flush(): void;
     isEmpty(): boolean;
-    set(pieces: SparseMultilineTokens[] | null, isComplete: boolean): void;
+    set(pieces: SparseMultilineTokens[] | null, isComplete: boolean, textModel?: ITextModel | undefined): void;
     setPartial(_range: Range, pieces: SparseMultilineTokens[]): Range;
     isComplete(): boolean;
     addSparseTokens(lineNumber: number, aTokens: LineTokens): LineTokens;

package/vscode/src/vs/editor/common/tokens/sparseTokensStore.js

@@ -16,9 +16,14 @@ class SparseTokensStore {
     isEmpty() {
         return (this._pieces.length === 0);
     }
-    set(pieces, isComplete) {
+    set(pieces, isComplete, textModel = undefined) {
         this._pieces = pieces || [];
         this._isComplete = isComplete;
+        if (textModel) {
+            for (const p of this._pieces) {
+                p.reportIfInvalid(textModel);
+            }
+        }
     }
     setPartial(_range, pieces) {
         let range = _range;
@@ -77,7 +82,7 @@ class SparseTokensStore {
         return this._isComplete;
     }
     addSparseTokens(lineNumber, aTokens) {
-        if (aTokens.getLineContent().length === 0) {
+        if (aTokens.getTextLength() === 0) {
             return aTokens;
         }
         const pieces = this._pieces;
@@ -104,8 +109,8 @@ class SparseTokensStore {
             result[resultLen++] = metadata;
         };
         for (let bIndex = 0; bIndex < bLen; bIndex++) {
-            const bStartCharacter = bTokens.getStartCharacter(bIndex);
-            const bEndCharacter = bTokens.getEndCharacter(bIndex);
+            const bStartCharacter = Math.min(bTokens.getStartCharacter(bIndex), aTokens.getTextLength());
+            const bEndCharacter = Math.min(bTokens.getEndCharacter(bIndex), aTokens.getTextLength());
             const bMetadata = bTokens.getMetadata(bIndex);
             const bMask = (((bMetadata & MetadataConsts.SEMANTIC_USE_ITALIC) ? MetadataConsts.ITALIC_MASK : 0)
                 | ((bMetadata & MetadataConsts.SEMANTIC_USE_BOLD) ? MetadataConsts.BOLD_MASK : 0)

package/vscode/src/vs/editor/common/tokens/tokenArray.d.ts

@@ -10,6 +10,7 @@ export declare class TokenArray {
     forEach(cb: (range: OffsetRange, tokenInfo: TokenInfo) => void): void;
     map<T>(cb: (range: OffsetRange, tokenInfo: TokenInfo) => T): T[];
     slice(range: OffsetRange): TokenArray;
+    append(other: TokenArray): TokenArray;
 }
 export type TokenMetadata = number;
 export declare class TokenInfo {

package/vscode/src/vs/editor/common/tokens/tokenArray.js

@@ -58,6 +58,10 @@ class TokenArray {
         }
         return TokenArray.create(result);
     }
+    append(other) {
+        const result = this._tokenInfo.concat(other._tokenInfo);
+        return TokenArray.create(result);
+    }
 }
 class TokenInfo {
     constructor(length, metadata) {

package/vscode/src/vs/editor/common/tokens/tokenWithTextArray.d.ts

@@ -0,0 +1,25 @@
+import { OffsetRange } from "../core/offsetRange.js";
+import { ILanguageIdCodec } from "../languages.js";
+import { LineTokens } from "./lineTokens.js";
+export declare class TokenWithTextArray {
+    private readonly _tokenInfo;
+    static fromLineTokens(lineTokens: LineTokens): TokenWithTextArray;
+    static create(tokenInfo: TokenWithTextInfo[]): TokenWithTextArray;
+    private constructor();
+    toLineTokens(decoder: ILanguageIdCodec): LineTokens;
+    forEach(cb: (range: OffsetRange, tokenInfo: TokenWithTextInfo) => void): void;
+    map<T>(cb: (range: OffsetRange, tokenInfo: TokenWithTextInfo) => T): T[];
+    slice(range: OffsetRange): TokenWithTextArray;
+    append(other: TokenWithTextArray): TokenWithTextArray;
+}
+export type TokenMetadata = number;
+export declare class TokenWithTextInfo {
+    readonly text: string;
+    readonly metadata: TokenMetadata;
+    constructor(text: string, metadata: TokenMetadata);
+}
+export declare class TokenWithTextArrayBuilder {
+    private readonly _tokens;
+    add(text: string, metadata: TokenMetadata): void;
+    build(): TokenWithTextArray;
+}

package/vscode/src/vs/editor/common/tokens/tokenWithTextArray.js

@@ -0,0 +1,73 @@
+
+import { OffsetRange } from '../core/offsetRange.js';
+import { LineTokens } from './lineTokens.js';
+
+class TokenWithTextArray {
+    static fromLineTokens(lineTokens) {
+        const tokenInfo = [];
+        for (let i = 0; i < lineTokens.getCount(); i++) {
+            tokenInfo.push(( new TokenWithTextInfo(lineTokens.getTokenText(i), lineTokens.getMetadata(i))));
+        }
+        return TokenWithTextArray.create(tokenInfo);
+    }
+    static create(tokenInfo) {
+        return ( new TokenWithTextArray(tokenInfo));
+    }
+    constructor(_tokenInfo) {
+        this._tokenInfo = _tokenInfo;
+    }
+    toLineTokens(decoder) {
+        return LineTokens.createFromTextAndMetadata(( this.map((_r, t) => ({ text: t.text, metadata: t.metadata }))), decoder);
+    }
+    forEach(cb) {
+        let lengthSum = 0;
+        for (const tokenInfo of this._tokenInfo) {
+            const range = ( new OffsetRange(lengthSum, lengthSum + tokenInfo.text.length));
+            cb(range, tokenInfo);
+            lengthSum += tokenInfo.text.length;
+        }
+    }
+    map(cb) {
+        const result = [];
+        let lengthSum = 0;
+        for (const tokenInfo of this._tokenInfo) {
+            const range = ( new OffsetRange(lengthSum, lengthSum + tokenInfo.text.length));
+            result.push(cb(range, tokenInfo));
+            lengthSum += tokenInfo.text.length;
+        }
+        return result;
+    }
+    slice(range) {
+        const result = [];
+        let lengthSum = 0;
+        for (const tokenInfo of this._tokenInfo) {
+            const tokenStart = lengthSum;
+            const tokenEndEx = tokenStart + tokenInfo.text.length;
+            if (tokenEndEx > range.start) {
+                if (tokenStart >= range.endExclusive) {
+                    break;
+                }
+                const deltaBefore = Math.max(0, range.start - tokenStart);
+                const deltaAfter = Math.max(0, tokenEndEx - range.endExclusive);
+                result.push(( new TokenWithTextInfo(
+                    tokenInfo.text.slice(deltaBefore, tokenInfo.text.length - deltaAfter),
+                    tokenInfo.metadata
+                )));
+            }
+            lengthSum += tokenInfo.text.length;
+        }
+        return TokenWithTextArray.create(result);
+    }
+    append(other) {
+        const result = this._tokenInfo.concat(other._tokenInfo);
+        return TokenWithTextArray.create(result);
+    }
+}
+class TokenWithTextInfo {
+    constructor(text, metadata) {
+        this.text = text;
+        this.metadata = metadata;
+    }
+}
+
+export { TokenWithTextArray, TokenWithTextInfo };