@codingame/monaco-vscode-api 17.2.1 → 18.0.1

package/vscode/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.js

@@ -0,0 +1,257 @@
+
+import { onUnexpectedError } from '../../../../base/common/errors.js';
+import { Emitter } from '../../../../base/common/event.js';
+import { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';
+import { countEOL } from '../../core/misc/eolCounter.js';
+import { Position } from '../../core/position.js';
+import { LineRange } from '../../core/ranges/lineRange.js';
+import { StandardTokenType } from '../../encodedTokenAttributes.js';
+import { TokenizationRegistry } from '../../languages.js';
+import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';
+import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';
+import { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';
+import { TrackingTokenizationStateStore, TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer } from '../textModelTokens.js';
+import { AbstractSyntaxTokenBackend, AttachedViewHandler } from './abstractSyntaxTokenBackend.js';
+
+class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
+    constructor(languageIdCodec, textModel, getLanguageId, attachedViews) {
+        super(languageIdCodec, textModel);
+        this.getLanguageId = getLanguageId;
+        this._tokenizer = null;
+        this._backgroundTokenizationState = BackgroundTokenizationState.InProgress;
+        this._onDidChangeBackgroundTokenizationState = this._register(( new Emitter()));
+        this.onDidChangeBackgroundTokenizationState = this._onDidChangeBackgroundTokenizationState.event;
+        this._defaultBackgroundTokenizer = null;
+        this._backgroundTokenizer = this._register(( new MutableDisposable()));
+        this._tokens = ( new ContiguousTokensStore(this._languageIdCodec));
+        this._debugBackgroundTokenizer = this._register(( new MutableDisposable()));
+        this._attachedViewStates = this._register(( new DisposableMap()));
+        this._register(TokenizationRegistry.onDidChange((e) => {
+            const languageId = this.getLanguageId();
+            if (e.changedLanguages.indexOf(languageId) === -1) {
+                return;
+            }
+            this.todo_resetTokenization();
+        }));
+        this.todo_resetTokenization();
+        this._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {
+            if (state) {
+                let existing = this._attachedViewStates.get(view);
+                if (!existing) {
+                    existing = ( new AttachedViewHandler(() => this.refreshRanges(existing.lineRanges)));
+                    this._attachedViewStates.set(view, existing);
+                }
+                existing.handleStateChange(state);
+            }
+            else {
+                this._attachedViewStates.deleteAndDispose(view);
+            }
+        }));
+    }
+    todo_resetTokenization(fireTokenChangeEvent = true) {
+        this._tokens.flush();
+        this._debugBackgroundTokens?.flush();
+        if (this._debugBackgroundStates) {
+            this._debugBackgroundStates = ( new TrackingTokenizationStateStore(this._textModel.getLineCount()));
+        }
+        if (fireTokenChangeEvent) {
+            this._onDidChangeTokens.fire({
+                semanticTokensApplied: false,
+                ranges: [
+                    {
+                        fromLineNumber: 1,
+                        toLineNumber: this._textModel.getLineCount(),
+                    },
+                ],
+            });
+        }
+        const initializeTokenization = () => {
+            if (this._textModel.isTooLargeForTokenization()) {
+                return [null, null];
+            }
+            const tokenizationSupport = TokenizationRegistry.get(this.getLanguageId());
+            if (!tokenizationSupport) {
+                return [null, null];
+            }
+            let initialState;
+            try {
+                initialState = tokenizationSupport.getInitialState();
+            }
+            catch (e) {
+                onUnexpectedError(e);
+                return [null, null];
+            }
+            return [tokenizationSupport, initialState];
+        };
+        const [tokenizationSupport, initialState] = initializeTokenization();
+        if (tokenizationSupport && initialState) {
+            this._tokenizer = ( new TokenizerWithStateStoreAndTextModel(
+                this._textModel.getLineCount(),
+                tokenizationSupport,
+                this._textModel,
+                this._languageIdCodec
+            ));
+        }
+        else {
+            this._tokenizer = null;
+        }
+        this._backgroundTokenizer.clear();
+        this._defaultBackgroundTokenizer = null;
+        if (this._tokenizer) {
+            const b = {
+                setTokens: (tokens) => {
+                    this.setTokens(tokens);
+                },
+                backgroundTokenizationFinished: () => {
+                    if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
+                        return;
+                    }
+                    const newState = BackgroundTokenizationState.Completed;
+                    this._backgroundTokenizationState = newState;
+                    this._onDidChangeBackgroundTokenizationState.fire();
+                },
+                setEndState: (lineNumber, state) => {
+                    if (!this._tokenizer) {
+                        return;
+                    }
+                    const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
+                    if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
+                        this._tokenizer?.store.setEndState(lineNumber, state);
+                    }
+                },
+            };
+            if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
+                this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
+            }
+            if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {
+                this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
+                    ( new DefaultBackgroundTokenizer(this._tokenizer, b));
+                this._defaultBackgroundTokenizer.handleChanges();
+            }
+            if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
+                this._debugBackgroundTokens = ( new ContiguousTokensStore(this._languageIdCodec));
+                this._debugBackgroundStates = ( new TrackingTokenizationStateStore(this._textModel.getLineCount()));
+                this._debugBackgroundTokenizer.clear();
+                this._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {
+                    setTokens: (tokens) => {
+                        this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
+                    },
+                    backgroundTokenizationFinished() {
+                    },
+                    setEndState: (lineNumber, state) => {
+                        this._debugBackgroundStates?.setEndState(lineNumber, state);
+                    },
+                });
+            }
+            else {
+                this._debugBackgroundTokens = undefined;
+                this._debugBackgroundStates = undefined;
+                this._debugBackgroundTokenizer.value = undefined;
+            }
+        }
+        this.refreshAllVisibleLineTokens();
+    }
+    handleDidChangeAttached() {
+        this._defaultBackgroundTokenizer?.handleChanges();
+    }
+    handleDidChangeContent(e) {
+        if (e.isFlush) {
+            this.todo_resetTokenization(false);
+        }
+        else if (!e.isEolChange) {
+            for (const c of e.changes) {
+                const [eolCount, firstLineLength] = countEOL(c.text);
+                this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
+                this._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);
+            }
+            this._debugBackgroundStates?.acceptChanges(e.changes);
+            if (this._tokenizer) {
+                this._tokenizer.store.acceptChanges(e.changes);
+            }
+            this._defaultBackgroundTokenizer?.handleChanges();
+        }
+    }
+    setTokens(tokens) {
+        const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
+        if (changes.length > 0) {
+            this._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });
+        }
+        return { changes: changes };
+    }
+    refreshAllVisibleLineTokens() {
+        const ranges = LineRange.joinMany(( [...this._attachedViewStates].map(([_, s]) => s.lineRanges)));
+        this.refreshRanges(ranges);
+    }
+    refreshRanges(ranges) {
+        for (const range of ranges) {
+            this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
+        }
+    }
+    refreshRange(startLineNumber, endLineNumber) {
+        if (!this._tokenizer) {
+            return;
+        }
+        startLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));
+        endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
+        const builder = ( new ContiguousMultilineTokensBuilder());
+        const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
+        const changedTokens = this.setTokens(builder.finalize());
+        if (heuristicTokens) {
+            for (const c of changedTokens.changes) {
+                this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
+            }
+        }
+        this._defaultBackgroundTokenizer?.checkFinished();
+    }
+    forceTokenization(lineNumber) {
+        const builder = ( new ContiguousMultilineTokensBuilder());
+        this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
+        this.setTokens(builder.finalize());
+        this._defaultBackgroundTokenizer?.checkFinished();
+    }
+    hasAccurateTokensForLine(lineNumber) {
+        if (!this._tokenizer) {
+            return true;
+        }
+        return this._tokenizer.hasAccurateTokensForLine(lineNumber);
+    }
+    isCheapToTokenize(lineNumber) {
+        if (!this._tokenizer) {
+            return true;
+        }
+        return this._tokenizer.isCheapToTokenize(lineNumber);
+    }
+    getLineTokens(lineNumber) {
+        const lineText = this._textModel.getLineContent(lineNumber);
+        const result = this._tokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
+        if (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {
+            if (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {
+                const backgroundResult = this._debugBackgroundTokens.getTokens(this._textModel.getLanguageId(), lineNumber - 1, lineText);
+                if (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {
+                    this._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);
+                }
+            }
+        }
+        return result;
+    }
+    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
+        if (!this._tokenizer) {
+            return StandardTokenType.Other;
+        }
+        const position = this._textModel.validatePosition(( new Position(lineNumber, column)));
+        this.forceTokenization(position.lineNumber);
+        return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
+    }
+    tokenizeLinesAt(lineNumber, lines) {
+        if (!this._tokenizer) {
+            return null;
+        }
+        this.forceTokenization(lineNumber);
+        return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
+    }
+    get hasTokens() {
+        return this._tokens.hasTokens;
+    }
+}
+
+export { TokenizerSyntaxTokenBackend };

package/vscode/src/vs/editor/common/model/tokens/treeSitter/cursorUtils.d.ts

@@ -0,0 +1,6 @@
+import type * as TreeSitter from "@vscode/tree-sitter-wasm";
+export declare function gotoNextSibling(newCursor: TreeSitter.TreeCursor, oldCursor: TreeSitter.TreeCursor): boolean;
+export declare function gotoParent(newCursor: TreeSitter.TreeCursor, oldCursor: TreeSitter.TreeCursor): boolean;
+export declare function gotoNthChild(newCursor: TreeSitter.TreeCursor, oldCursor: TreeSitter.TreeCursor, index: number): boolean;
+export declare function nextSiblingOrParentSibling(newCursor: TreeSitter.TreeCursor, oldCursor: TreeSitter.TreeCursor): boolean;
+export declare function getClosestPreviousNodes(cursor: TreeSitter.TreeCursor, tree: TreeSitter.Tree): TreeSitter.Node | undefined;

package/vscode/src/vs/editor/common/model/tokens/treeSitter/cursorUtils.js

@@ -0,0 +1,76 @@
+
+
+function gotoNextSibling(newCursor, oldCursor) {
+    const n = newCursor.gotoNextSibling();
+    const o = oldCursor.gotoNextSibling();
+    if (n !== o) {
+        throw ( new Error('Trees are out of sync'));
+    }
+    return n && o;
+}
+function gotoParent(newCursor, oldCursor) {
+    const n = newCursor.gotoParent();
+    const o = oldCursor.gotoParent();
+    if (n !== o) {
+        throw ( new Error('Trees are out of sync'));
+    }
+    return n && o;
+}
+function gotoNthChild(newCursor, oldCursor, index) {
+    const n = newCursor.gotoFirstChild();
+    const o = oldCursor.gotoFirstChild();
+    if (n !== o) {
+        throw ( new Error('Trees are out of sync'));
+    }
+    if (index === 0) {
+        return n && o;
+    }
+    for (let i = 1; i <= index; i++) {
+        const nn = newCursor.gotoNextSibling();
+        const oo = oldCursor.gotoNextSibling();
+        if (nn !== oo) {
+            throw ( new Error('Trees are out of sync'));
+        }
+        if (!nn || !oo) {
+            return false;
+        }
+    }
+    return n && o;
+}
+function nextSiblingOrParentSibling(newCursor, oldCursor) {
+    do {
+        if (newCursor.currentNode.nextSibling) {
+            return gotoNextSibling(newCursor, oldCursor);
+        }
+        if (newCursor.currentNode.parent) {
+            gotoParent(newCursor, oldCursor);
+        }
+    } while (newCursor.currentNode.nextSibling || newCursor.currentNode.parent);
+    return false;
+}
+function getClosestPreviousNodes(cursor, tree) {
+    const findPrev = tree.walk();
+    findPrev.resetTo(cursor);
+    const startingNode = cursor.currentNode;
+    do {
+        if (findPrev.currentNode.previousSibling && ((findPrev.currentNode.endIndex - findPrev.currentNode.startIndex) !== 0)) {
+            findPrev.gotoPreviousSibling();
+        }
+        else {
+            while (!findPrev.currentNode.previousSibling && findPrev.currentNode.parent) {
+                findPrev.gotoParent();
+            }
+            findPrev.gotoPreviousSibling();
+        }
+    } while ((findPrev.currentNode.endIndex > startingNode.startIndex)
+        && (findPrev.currentNode.parent || findPrev.currentNode.previousSibling)
+        && (findPrev.currentNode.id !== startingNode.id));
+    if ((findPrev.currentNode.id !== startingNode.id) && findPrev.currentNode.endIndex <= startingNode.startIndex) {
+        return findPrev.currentNode;
+    }
+    else {
+        return undefined;
+    }
+}
+
+export { getClosestPreviousNodes, gotoNextSibling, gotoNthChild, gotoParent, nextSiblingOrParentSibling };

package/vscode/src/vs/editor/common/model/tokens/treeSitter/tokenStore.d.ts

@@ -0,0 +1,65 @@
+import { IDisposable } from "../../../../../base/common/lifecycle.js";
+import { ITextModel } from "../../../model.js";
+declare class ListNode implements IDisposable {
+    readonly height: number;
+    parent?: ListNode;
+    private readonly _children;
+    get children(): ReadonlyArray<Node>;
+    private _length;
+    get length(): number;
+    constructor(height: number);
+    static create(node1: Node, node2: Node): ListNode;
+    canAppendChild(): boolean;
+    appendChild(node: Node): void;
+    private _updateParentLength;
+    unappendChild(): Node;
+    prependChild(node: Node): void;
+    unprependChild(): Node;
+    lastChild(): Node;
+    dispose(): void;
+}
+export declare enum TokenQuality {
+    None = 0,
+    ViewportGuess = 1,
+    EditGuess = 2,
+    Accurate = 3
+}
+type Node = ListNode | LeafNode;
+interface LeafNode {
+    readonly length: number;
+    token: number;
+    tokenQuality: TokenQuality;
+    height: 0;
+}
+export interface TokenUpdate {
+    readonly startOffsetInclusive: number;
+    readonly length: number;
+    readonly token: number;
+}
+export declare class TokenStore implements IDisposable {
+    private readonly _textModel;
+    private _root;
+    get root(): Node;
+    constructor(_textModel: ITextModel);
+    private createEmptyRoot;
+    buildStore(tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
+    private createFromUpdates;
+    update(length: number, tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
+    delete(length: number, startOffset: number): void;
+    private replace;
+    private traverseInOrderInRange;
+    getTokenAt(offset: number): TokenUpdate | undefined;
+    getTokensInRange(startOffsetInclusive: number, endOffsetExclusive: number): TokenUpdate[];
+    markForRefresh(startOffsetInclusive: number, endOffsetExclusive: number): void;
+    rangeHasTokens(startOffsetInclusive: number, endOffsetExclusive: number, minimumTokenQuality: TokenQuality): boolean;
+    rangeNeedsRefresh(startOffsetInclusive: number, endOffsetExclusive: number): boolean;
+    getNeedsRefresh(): {
+        startOffset: number;
+        endOffset: number;
+    }[];
+    deepCopy(): TokenStore;
+    private _copyNodeIterative;
+    printTree(root?: Node): string;
+    dispose(): void;
+}
+export {};