@codingame/monaco-vscode-api 13.1.7 → 14.0.1

package/vscode/src/vs/editor/common/model/tokenStore.js

@@ -0,0 +1,384 @@
+
+
+class ListNode {
+    get children() { return this._children; }
+    get length() { return this._length; }
+    constructor(height) {
+        this.height = height;
+        this._children = [];
+        this._length = 0;
+    }
+    static create(node1, node2) {
+        const list = ( new ListNode(node1.height + 1));
+        list.appendChild(node1);
+        list.appendChild(node2);
+        return list;
+    }
+    canAppendChild() {
+        return this._children.length < 3;
+    }
+    appendChild(node) {
+        if (!this.canAppendChild()) {
+            throw ( new Error('Cannot insert more than 3 children in a ListNode'));
+        }
+        this._children.push(node);
+        this._length += node.length;
+        this._updateParentLength(node.length);
+        node.parent = this;
+    }
+    _updateParentLength(delta) {
+        let updateParent = this.parent;
+        while (updateParent) {
+            updateParent._length += delta;
+            updateParent = updateParent.parent;
+        }
+    }
+    unappendChild() {
+        const child = this._children.pop();
+        this._length -= child.length;
+        this._updateParentLength(-child.length);
+        return child;
+    }
+    prependChild(node) {
+        if (this._children.length >= 3) {
+            throw ( new Error('Cannot prepend more than 3 children in a ListNode'));
+        }
+        this._children.unshift(node);
+        this._length += node.length;
+        this._updateParentLength(node.length);
+        node.parent = this;
+    }
+    unprependChild() {
+        const child = this._children.shift();
+        this._length -= child.length;
+        this._updateParentLength(-child.length);
+        return child;
+    }
+    lastChild() {
+        return this._children[this._children.length - 1];
+    }
+    dispose() {
+        this._children.splice(0, this._children.length);
+    }
+}
+function isLeaf(node) {
+    return node.token !== undefined;
+}
+function append(node, nodeToAppend) {
+    let curNode = node;
+    const parents = [];
+    let nodeToAppendOfCorrectHeight;
+    while (true) {
+        if (nodeToAppend.height === curNode.height) {
+            nodeToAppendOfCorrectHeight = nodeToAppend;
+            break;
+        }
+        if (isLeaf(curNode)) {
+            throw ( new Error('unexpected'));
+        }
+        parents.push(curNode);
+        curNode = curNode.lastChild();
+    }
+    for (let i = parents.length - 1; i >= 0; i--) {
+        const parent = parents[i];
+        if (nodeToAppendOfCorrectHeight) {
+            if (parent.children.length >= 3) {
+                const newList = ListNode.create(parent.unappendChild(), nodeToAppendOfCorrectHeight);
+                nodeToAppendOfCorrectHeight = newList;
+            }
+            else {
+                parent.appendChild(nodeToAppendOfCorrectHeight);
+                nodeToAppendOfCorrectHeight = undefined;
+            }
+        }
+    }
+    if (nodeToAppendOfCorrectHeight) {
+        const newList = ( new ListNode(nodeToAppendOfCorrectHeight.height + 1));
+        newList.appendChild(node);
+        newList.appendChild(nodeToAppendOfCorrectHeight);
+        return newList;
+    }
+    else {
+        return node;
+    }
+}
+function prepend(list, nodeToAppend) {
+    let curNode = list;
+    const parents = [];
+    while (nodeToAppend.height !== curNode.height) {
+        if (isLeaf(curNode)) {
+            throw ( new Error('unexpected'));
+        }
+        parents.push(curNode);
+        curNode = curNode.children[0];
+    }
+    let nodeToPrependOfCorrectHeight = nodeToAppend;
+    for (let i = parents.length - 1; i >= 0; i--) {
+        const parent = parents[i];
+        if (nodeToPrependOfCorrectHeight) {
+            if (parent.children.length >= 3) {
+                nodeToPrependOfCorrectHeight = ListNode.create(nodeToPrependOfCorrectHeight, parent.unprependChild());
+            }
+            else {
+                parent.prependChild(nodeToPrependOfCorrectHeight);
+                nodeToPrependOfCorrectHeight = undefined;
+            }
+        }
+    }
+    if (nodeToPrependOfCorrectHeight) {
+        return ListNode.create(nodeToPrependOfCorrectHeight, list);
+    }
+    else {
+        return list;
+    }
+}
+function concat(node1, node2) {
+    if (node1.height === node2.height) {
+        return ListNode.create(node1, node2);
+    }
+    else if (node1.height > node2.height) {
+        return append(node1, node2);
+    }
+    else {
+        return prepend(node2, node1);
+    }
+}
+class TokenStore {
+    get root() {
+        return this._root;
+    }
+    constructor(_textModel) {
+        this._textModel = _textModel;
+        this._root = this.createEmptyRoot();
+    }
+    createEmptyRoot() {
+        return {
+            length: this._textModel.getValueLength(),
+            token: 0,
+            height: 0
+        };
+    }
+    buildStore(tokens) {
+        this._root = this.createFromUpdates(tokens, true);
+    }
+    createFromUpdates(tokens, needsRefresh) {
+        if (tokens.length === 0) {
+            return this.createEmptyRoot();
+        }
+        let newRoot = {
+            length: tokens[0].length,
+            token: tokens[0].token,
+            height: 0,
+            needsRefresh
+        };
+        for (let j = 1; j < tokens.length; j++) {
+            newRoot = append(newRoot, { length: tokens[j].length, token: tokens[j].token, height: 0, needsRefresh });
+        }
+        return newRoot;
+    }
+    update(length, tokens, needsRefresh) {
+        if (tokens.length === 0) {
+            return;
+        }
+        this.replace(length, tokens[0].startOffsetInclusive, tokens, needsRefresh);
+    }
+    delete(length, startOffset) {
+        this.replace(length, startOffset, [], true);
+    }
+    replace(length, updateOffsetStart, tokens, needsRefresh) {
+        const firstUnchangedOffsetAfterUpdate = updateOffsetStart + length;
+        const precedingNodes = [];
+        const postcedingNodes = [];
+        const stack = [{ node: this._root, offset: 0 }];
+        while (stack.length > 0) {
+            const node = stack.pop();
+            const currentOffset = node.offset;
+            if (currentOffset < updateOffsetStart && currentOffset + node.node.length <= updateOffsetStart) {
+                node.node.parent = undefined;
+                precedingNodes.push(node.node);
+                continue;
+            }
+            else if (isLeaf(node.node) && (currentOffset < updateOffsetStart)) {
+                precedingNodes.push({ length: updateOffsetStart - currentOffset, token: node.node.token, height: 0, needsRefresh: needsRefresh || node.node.needsRefresh });
+            }
+            if ((updateOffsetStart <= currentOffset) && (currentOffset + node.node.length <= firstUnchangedOffsetAfterUpdate)) {
+                continue;
+            }
+            if (currentOffset >= firstUnchangedOffsetAfterUpdate) {
+                node.node.parent = undefined;
+                postcedingNodes.push(node.node);
+                continue;
+            }
+            else if (isLeaf(node.node) && (currentOffset + node.node.length >= firstUnchangedOffsetAfterUpdate)) {
+                postcedingNodes.push({ length: currentOffset + node.node.length - firstUnchangedOffsetAfterUpdate, token: node.node.token, height: 0, needsRefresh: needsRefresh || node.node.needsRefresh });
+                continue;
+            }
+            if (!isLeaf(node.node)) {
+                let childOffset = currentOffset + node.node.length;
+                for (let i = node.node.children.length - 1; i >= 0; i--) {
+                    childOffset -= node.node.children[i].length;
+                    stack.push({ node: node.node.children[i], offset: childOffset });
+                }
+            }
+        }
+        let allNodes;
+        if (tokens.length > 0) {
+            allNodes = precedingNodes.concat(this.createFromUpdates(tokens, needsRefresh), postcedingNodes);
+        }
+        else {
+            allNodes = precedingNodes.concat(postcedingNodes);
+        }
+        let newRoot = allNodes[0];
+        for (let i = 1; i < allNodes.length; i++) {
+            newRoot = concat(newRoot, allNodes[i]);
+        }
+        this._root = newRoot ?? this.createEmptyRoot();
+    }
+    traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, visitor) {
+        const stack = [{ node: this._root, offset: 0 }];
+        while (stack.length > 0) {
+            const { node, offset } = stack.pop();
+            const nodeEnd = offset + node.length;
+            if (nodeEnd <= startOffsetInclusive || offset >= endOffsetExclusive) {
+                continue;
+            }
+            if (visitor(node, offset)) {
+                return;
+            }
+            if (!isLeaf(node)) {
+                let childOffset = offset + node.length;
+                for (let i = node.children.length - 1; i >= 0; i--) {
+                    childOffset -= node.children[i].length;
+                    stack.push({ node: node.children[i], offset: childOffset });
+                }
+            }
+        }
+    }
+    getTokenAt(offset) {
+        let result;
+        this.traverseInOrderInRange(offset, this._root.length, (node, offset) => {
+            if (isLeaf(node)) {
+                result = { token: node.token, startOffsetInclusive: offset, length: node.length };
+                return true;
+            }
+            return false;
+        });
+        return result;
+    }
+    getTokensInRange(startOffsetInclusive, endOffsetExclusive) {
+        const result = [];
+        this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node, offset) => {
+            if (isLeaf(node)) {
+                let clippedLength = node.length;
+                let clippedOffset = offset;
+                if (offset < startOffsetInclusive) {
+                    clippedLength -= (startOffsetInclusive - offset);
+                    clippedOffset = startOffsetInclusive;
+                }
+                else if (offset + node.length > endOffsetExclusive) {
+                    clippedLength -= (offset + node.length - endOffsetExclusive);
+                }
+                result.push({ token: node.token, startOffsetInclusive: clippedOffset, length: clippedLength });
+            }
+            return false;
+        });
+        return result;
+    }
+    markForRefresh(startOffsetInclusive, endOffsetExclusive) {
+        this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node) => {
+            if (isLeaf(node)) {
+                node.needsRefresh = true;
+            }
+            return false;
+        });
+    }
+    rangeNeedsRefresh(startOffsetInclusive, endOffsetExclusive) {
+        let needsRefresh = false;
+        this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node) => {
+            if (isLeaf(node) && node.needsRefresh) {
+                needsRefresh = true;
+            }
+            return false;
+        });
+        return needsRefresh;
+    }
+    getNeedsRefresh() {
+        const result = [];
+        this.traverseInOrderInRange(0, this._textModel.getValueLength(), (node, offset) => {
+            if (isLeaf(node) && node.needsRefresh) {
+                if ((result.length > 0) && (result[result.length - 1].endOffset === offset)) {
+                    result[result.length - 1].endOffset += node.length;
+                }
+                else {
+                    result.push({ startOffset: offset, endOffset: offset + node.length });
+                }
+            }
+            return false;
+        });
+        return result;
+    }
+    deepCopy() {
+        const newStore = ( new TokenStore(this._textModel));
+        newStore._root = this._copyNodeIterative(this._root);
+        return newStore;
+    }
+    _copyNodeIterative(root) {
+        const newRoot = isLeaf(root)
+            ? { length: root.length, token: root.token, needsRefresh: root.needsRefresh, height: root.height }
+            : ( new ListNode(root.height));
+        const stack = [[root, newRoot]];
+        while (stack.length > 0) {
+            const [oldNode, clonedNode] = stack.pop();
+            if (!isLeaf(oldNode)) {
+                for (const child of oldNode.children) {
+                    const childCopy = isLeaf(child)
+                        ? { length: child.length, token: child.token, needsRefresh: child.needsRefresh, height: child.height }
+                        : ( new ListNode(child.height));
+                    clonedNode.appendChild(childCopy);
+                    stack.push([child, childCopy]);
+                }
+            }
+        }
+        return newRoot;
+    }
+    printTree(root = this._root) {
+        const result = [];
+        const stack = [[root, 0]];
+        while (stack.length > 0) {
+            const [node, depth] = stack.pop();
+            const indent = '  '.repeat(depth);
+            if (isLeaf(node)) {
+                result.push(`${indent}Leaf(length: ${node.length}, token: ${node.token}, refresh: ${node.needsRefresh})\n`);
+            }
+            else {
+                result.push(`${indent}List(length: ${node.length})\n`);
+                for (let i = node.children.length - 1; i >= 0; i--) {
+                    stack.push([node.children[i], depth + 1]);
+                }
+            }
+        }
+        return result.join('');
+    }
+    dispose() {
+        const stack = [[this._root, false]];
+        while (stack.length > 0) {
+            const [node, visited] = stack.pop();
+            if (isLeaf(node)) {
+                node.parent = undefined;
+            }
+            else if (!visited) {
+                stack.push([node, true]);
+                for (let i = node.children.length - 1; i >= 0; i--) {
+                    stack.push([node.children[i], false]);
+                }
+            }
+            else {
+                node.dispose();
+                node.parent = undefined;
+            }
+        }
+        this._root = undefined;
+    }
+}
+
+export { TokenStore };

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.d.ts

@@ -9,11 +9,11 @@ import { BracketPairsTextModelPart } from "./bracketPairsTextModelPart/bracketPa
 import { TextModel } from "./textModel.js";
 import { TextModelPart } from "./textModelPart.js";
 import { AttachedViews } from "./tokens.js";
-import { ITreeSitterParserService } from "../services/treeSitterParserService.js";
 import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from "../textModelEvents.js";
-import { BackgroundTokenizationState, ITokenizationTextModelPart, ITokenizeLineWithEditResult, LineEditWithAdditionalLines } from "../tokenizationTextModelPart.js";
+import { BackgroundTokenizationState, ITokenizationTextModelPart } from "../tokenizationTextModelPart.js";
 import { LineTokens } from "../tokens/lineTokens.js";
 import { SparseMultilineTokens } from "../tokens/sparseMultilineTokens.js";
+import { IInstantiationService } from "../../../platform/instantiation/common/instantiation.js";
 export declare class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {
     private readonly _textModel;
     private readonly _bracketPairsTextModelPart;
@@ -21,7 +21,7 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     private readonly _attachedViews;
     private readonly _languageService;
     private readonly _languageConfigurationService;
-    private readonly _treeSitterService;
+    private readonly _instantiationService;
     private readonly _semanticTokens;
     private readonly _onDidChangeLanguage;
     readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;
@@ -31,7 +31,7 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;
     private _tokens;
     private readonly _tokensDisposables;
-    constructor(_textModel: TextModel, _bracketPairsTextModelPart: BracketPairsTextModelPart, _languageId: string, _attachedViews: AttachedViews, _languageService: ILanguageService, _languageConfigurationService: ILanguageConfigurationService, _treeSitterService: ITreeSitterParserService);
+    constructor(_textModel: TextModel, _bracketPairsTextModelPart: BracketPairsTextModelPart, _languageId: string, _attachedViews: AttachedViews, _languageService: ILanguageService, _languageConfigurationService: ILanguageConfigurationService, _instantiationService: IInstantiationService);
     private createGrammarTokens;
     private createTreeSitterTokens;
     private createTokens;
@@ -51,7 +51,7 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     isCheapToTokenize(lineNumber: number): boolean;
     tokenizeIfCheap(lineNumber: number): void;
     getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
-    tokenizeLineWithEdit(lineNumber: number, edit: LineEditWithAdditionalLines): ITokenizeLineWithEditResult;
+    tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
     setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void;
     hasCompleteSemanticTokens(): boolean;
     hasSomeSemanticTokens(): boolean;

package/vscode/src/vs/editor/common/model/tokenizationTextModelPart.js

@@ -16,15 +16,15 @@ import { TextModelPart } from './textModelPart.js';
 import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from './textModelTokens.js';
 import { AbstractTokens, AttachedViewHandler } from './tokens.js';
 import { TreeSitterTokens } from './treeSitterTokens.js';
-import { ITreeSitterParserService } from '../services/treeSitterParserService.js';
 import { BackgroundTokenizationState } from '../tokenizationTextModelPart.js';
 import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
 import { ContiguousTokensStore } from '../tokens/contiguousTokensStore.js';
 import { SparseTokensStore } from '../tokens/sparseTokensStore.js';
+import { IInstantiationService } from '../../../platform/instantiation/common/instantiation.js';
 
 var TokenizationTextModelPart_1;
 let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
-    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _treeSitterService) {
+    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _instantiationService) {
         super();
         this._textModel = _textModel;
         this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
@@ -32,7 +32,7 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
         this._attachedViews = _attachedViews;
         this._languageService = _languageService;
         this._languageConfigurationService = _languageConfigurationService;
-        this._treeSitterService = _treeSitterService;
+        this._instantiationService = _instantiationService;
         this._semanticTokens = ( new SparseTokensStore(this._languageService.languageIdCodec));
         this._onDidChangeLanguage = this._register(( new Emitter()));
         this.onDidChangeLanguage = this._onDidChangeLanguage.event;
@@ -55,12 +55,7 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
         )));
     }
     createTreeSitterTokens() {
-        return this._register(( new TreeSitterTokens(
-            this._treeSitterService,
-            this._languageService.languageIdCodec,
-            this._textModel,
-            () => this._languageId
-        )));
+        return this._register(this._instantiationService.createInstance(TreeSitterTokens, this._languageService.languageIdCodec, this._textModel, () => this._languageId));
     }
     createTokens(useTreeSitter) {
         const needsReset = this._tokens !== undefined;
@@ -158,8 +153,8 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
     getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
         return this._tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
     }
-    tokenizeLineWithEdit(lineNumber, edit) {
-        return this._tokens.tokenizeLineWithEdit(lineNumber, edit);
+    tokenizeLinesAt(lineNumber, lines) {
+        return this._tokens.tokenizeLinesAt(lineNumber, lines);
     }
     setSemanticTokens(tokens, isComplete) {
         this._semanticTokens.set(tokens, isComplete);
@@ -267,7 +262,7 @@ let TokenizationTextModelPart = TokenizationTextModelPart_1 = class Tokenization
 TokenizationTextModelPart = TokenizationTextModelPart_1 = ( __decorate([
     ( __param(4, ILanguageService)),
     ( __param(5, ILanguageConfigurationService)),
-    ( __param(6, ITreeSitterParserService))
+    ( __param(6, IInstantiationService))
 ], TokenizationTextModelPart));
 class GrammarTokens extends AbstractTokens {
     constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
@@ -494,12 +489,12 @@ class GrammarTokens extends AbstractTokens {
         this.forceTokenization(position.lineNumber);
         return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
     }
-    tokenizeLineWithEdit(lineNumber, edit) {
+    tokenizeLinesAt(lineNumber, lines) {
         if (!this._tokenizer) {
-            return { mainLineTokens: null, additionalLines: null };
+            return null;
         }
         this.forceTokenization(lineNumber);
-        return this._tokenizer.tokenizeLineWithEdit(lineNumber, edit);
+        return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
     }
     get hasTokens() {
         return this._tokens.hasTokens;

package/vscode/src/vs/editor/common/model/tokens.d.ts

@@ -6,7 +6,7 @@ import { ILanguageIdCodec } from "../languages.js";
 import { IAttachedView } from "../model.js";
 import { TextModel } from "./textModel.js";
 import { IModelContentChangedEvent, IModelTokensChangedEvent } from "../textModelEvents.js";
-import { BackgroundTokenizationState, ITokenizeLineWithEditResult, LineEditWithAdditionalLines } from "../tokenizationTextModelPart.js";
+import { BackgroundTokenizationState } from "../tokenizationTextModelPart.js";
 import { LineTokens } from "../tokens/lineTokens.js";
 export declare class AttachedViews {
     private readonly _onDidChangeVisibleRanges;
@@ -52,6 +52,6 @@ export declare abstract class AbstractTokens extends Disposable {
     tokenizeIfCheap(lineNumber: number): void;
     abstract getLineTokens(lineNumber: number): LineTokens;
     abstract getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
-    abstract tokenizeLineWithEdit(lineNumber: number, edit: LineEditWithAdditionalLines): ITokenizeLineWithEditResult;
+    abstract tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
     abstract get hasTokens(): boolean;
 }

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.d.ts

@@ -0,0 +1,44 @@
+import { Range } from "../core/range.js";
+import { ITextModel } from "../model.js";
+import { TokenUpdate } from "./tokenStore.js";
+import { IDisposable } from "../../../base/common/lifecycle.js";
+export interface ITreeSitterTokenizationStoreService {
+    readonly _serviceBrand: undefined;
+    setTokens(model: ITextModel, tokens: TokenUpdate[]): void;
+    getTokens(model: ITextModel, line: number): Uint32Array | undefined;
+    updateTokens(model: ITextModel, version: number, updates: {
+        oldRangeLength: number;
+        newTokens: TokenUpdate[];
+    }[]): void;
+    markForRefresh(model: ITextModel, range: Range): void;
+    getNeedsRefresh(model: ITextModel): {
+        range: Range;
+        startOffset: number;
+        endOffset: number;
+    }[];
+    hasTokens(model: ITextModel, accurateForRange?: Range): boolean;
+}
+export declare const ITreeSitterTokenizationStoreService: import("../../../platform/instantiation/common/instantiation.js").ServiceIdentifier<ITreeSitterTokenizationStoreService>;
+export interface TokenInformation {
+    tokens: Uint32Array;
+    needsRefresh?: boolean;
+}
+export declare class TreeSitterTokenizationStoreService implements ITreeSitterTokenizationStoreService, IDisposable {
+    readonly _serviceBrand: undefined;
+    private readonly tokens;
+    constructor();
+    setTokens(model: ITextModel, tokens: TokenUpdate[]): void;
+    hasTokens(model: ITextModel, accurateForRange?: Range): boolean;
+    getTokens(model: ITextModel, line: number): Uint32Array | undefined;
+    updateTokens(model: ITextModel, version: number, updates: {
+        oldRangeLength: number;
+        newTokens: TokenUpdate[];
+    }[]): void;
+    markForRefresh(model: ITextModel, range: Range): void;
+    getNeedsRefresh(model: ITextModel): {
+        range: Range;
+        startOffset: number;
+        endOffset: number;
+    }[];
+    dispose(): void;
+}

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.js

@@ -0,0 +1,114 @@
+
+import { Range } from '../core/range.js';
+import { TokenStore } from './tokenStore.js';
+import { registerSingleton, InstantiationType } from '../../../platform/instantiation/common/extensions.js';
+import { createDecorator } from '../../../platform/instantiation/common/instantiation.js';
+import { DisposableStore } from '../../../base/common/lifecycle.js';
+
+const ITreeSitterTokenizationStoreService = ( createDecorator('treeSitterTokenizationStoreService'));
+class TreeSitterTokenizationStoreService {
+    constructor() {
+        this.tokens = ( new Map());
+    }
+    setTokens(model, tokens) {
+        const disposables = ( new DisposableStore());
+        const store = disposables.add(( new TokenStore(model)));
+        this.tokens.set(model, { store: store, accurateVersion: model.getVersionId(), disposables, guessVersion: model.getVersionId() });
+        store.buildStore(tokens);
+        disposables.add(model.onDidChangeContent(e => {
+            const storeInfo = this.tokens.get(model);
+            if (!storeInfo) {
+                return;
+            }
+            storeInfo.guessVersion = e.versionId;
+            for (const change of e.changes) {
+                if (change.text.length > change.rangeLength) {
+                    const oldToken = storeInfo.store.getTokenAt(change.rangeOffset);
+                    let newToken;
+                    if (oldToken) {
+                        newToken = { startOffsetInclusive: oldToken.startOffsetInclusive, length: oldToken.length + change.text.length - change.rangeLength, token: oldToken.token };
+                    }
+                    else {
+                        newToken = { startOffsetInclusive: change.rangeOffset, length: change.text.length, token: 0 };
+                    }
+                    storeInfo.store.update(oldToken?.length ?? 0, [newToken], true);
+                }
+                else if (change.text.length < change.rangeLength) {
+                    const deletedCharCount = change.rangeLength - change.text.length;
+                    storeInfo.store.delete(deletedCharCount, change.rangeOffset);
+                }
+                const refreshLength = change.rangeLength > change.text.length ? change.rangeLength : change.text.length;
+                storeInfo.store.markForRefresh(change.rangeOffset, change.rangeOffset + refreshLength);
+            }
+        }));
+        disposables.add(model.onWillDispose(() => {
+            const storeInfo = this.tokens.get(model);
+            if (storeInfo) {
+                storeInfo.disposables.dispose();
+                this.tokens.delete(model);
+            }
+        }));
+    }
+    hasTokens(model, accurateForRange) {
+        const tokens = this.tokens.get(model);
+        if (!tokens) {
+            return false;
+        }
+        if (!accurateForRange || (tokens.guessVersion === tokens.accurateVersion)) {
+            return true;
+        }
+        return !tokens.store.rangeNeedsRefresh(model.getOffsetAt(accurateForRange.getStartPosition()), model.getOffsetAt(accurateForRange.getEndPosition()));
+    }
+    getTokens(model, line) {
+        const tokens = this.tokens.get(model)?.store;
+        if (!tokens) {
+            return undefined;
+        }
+        const lineStartOffset = model.getOffsetAt({ lineNumber: line, column: 1 });
+        const lineTokens = tokens.getTokensInRange(lineStartOffset, model.getOffsetAt({ lineNumber: line, column: model.getLineMaxColumn(line) }) + 1);
+        const result = ( new Uint32Array(lineTokens.length * 2));
+        for (let i = 0; i < lineTokens.length; i++) {
+            result[i * 2] = lineTokens[i].startOffsetInclusive - lineStartOffset + lineTokens[i].length;
+            result[i * 2 + 1] = lineTokens[i].token;
+        }
+        return result;
+    }
+    updateTokens(model, version, updates) {
+        const existingTokens = this.tokens.get(model);
+        if (!existingTokens) {
+            return;
+        }
+        existingTokens.accurateVersion = version;
+        for (const update of updates) {
+            const lastToken = update.newTokens.length > 0 ? update.newTokens[update.newTokens.length - 1] : undefined;
+            const oldRangeLength = ((existingTokens.guessVersion >= version) && lastToken) ? (lastToken.startOffsetInclusive + lastToken.length - update.newTokens[0].startOffsetInclusive) : update.oldRangeLength;
+            existingTokens.store.update(oldRangeLength, update.newTokens);
+        }
+    }
+    markForRefresh(model, range) {
+        const tree = this.tokens.get(model)?.store;
+        if (!tree) {
+            return;
+        }
+        tree.markForRefresh(model.getOffsetAt(range.getStartPosition()), model.getOffsetAt(range.getEndPosition()));
+    }
+    getNeedsRefresh(model) {
+        const needsRefreshOffsetRanges = this.tokens.get(model)?.store.getNeedsRefresh();
+        if (!needsRefreshOffsetRanges) {
+            return [];
+        }
+        return ( needsRefreshOffsetRanges.map(range => ({
+            range: Range.fromPositions(model.getPositionAt(range.startOffset), model.getPositionAt(range.endOffset)),
+            startOffset: range.startOffset,
+            endOffset: range.endOffset
+        })));
+    }
+    dispose() {
+        for (const [, value] of this.tokens) {
+            value.disposables.dispose();
+        }
+    }
+}
+registerSingleton(ITreeSitterTokenizationStoreService, TreeSitterTokenizationStoreService, InstantiationType.Delayed);
+
+export { ITreeSitterTokenizationStoreService, TreeSitterTokenizationStoreService };