@codingame/monaco-vscode-api 17.2.1 → 18.0.1

package/vscode/src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.js

@@ -0,0 +1,118 @@
+
+import { equals } from '../../../../base/common/arrays.js';
+import { RunOnceScheduler } from '../../../../base/common/async.js';
+import { Emitter } from '../../../../base/common/event.js';
+import { Disposable } from '../../../../base/common/lifecycle.js';
+import { LineRange } from '../../core/ranges/lineRange.js';
+import '../../../../base/common/observableInternal/index.js';
+import { itemsEquals, itemEquals, equalsIfDefined } from '../../../../base/common/equals.js';
+import { observableSignal } from '../../../../base/common/observableInternal/observables/observableSignal.js';
+import { derivedOpts } from '../../../../base/common/observableInternal/observables/derived.js';
+import { observableValueOpts } from '../../../../base/common/observableInternal/observables/observableValueOpts.js';
+
+class AttachedViews {
+    constructor() {
+        this._onDidChangeVisibleRanges = ( new Emitter());
+        this.onDidChangeVisibleRanges = this._onDidChangeVisibleRanges.event;
+        this._views = ( new Set());
+        this._viewsChanged = observableSignal(this);
+        this.visibleLineRanges = derivedOpts({
+            owner: this,
+            equalsFn: itemsEquals(itemEquals())
+        }, reader => {
+            this._viewsChanged.read(reader);
+            const ranges = LineRange.joinMany(( [...this._views].map(view => view.state.read(reader)?.visibleLineRanges ?? [])));
+            return ranges;
+        });
+    }
+    attachView() {
+        const view = ( new AttachedViewImpl((state) => {
+            this._onDidChangeVisibleRanges.fire({ view, state });
+        }));
+        this._views.add(view);
+        this._viewsChanged.trigger(undefined);
+        return view;
+    }
+    detachView(view) {
+        this._views.delete(view);
+        this._onDidChangeVisibleRanges.fire({ view, state: undefined });
+        this._viewsChanged.trigger(undefined);
+    }
+}
+class AttachedViewState {
+    constructor(visibleLineRanges, stabilized) {
+        this.visibleLineRanges = visibleLineRanges;
+        this.stabilized = stabilized;
+    }
+    equals(other) {
+        if (this === other) {
+            return true;
+        }
+        if (!equals(this.visibleLineRanges, other.visibleLineRanges, (a, b) => a.equals(b))) {
+            return false;
+        }
+        if (this.stabilized !== other.stabilized) {
+            return false;
+        }
+        return true;
+    }
+}
+class AttachedViewImpl {
+    get state() { return this._state; }
+    constructor(handleStateChange) {
+        this.handleStateChange = handleStateChange;
+        this._state = observableValueOpts({ owner: this, equalsFn: equalsIfDefined((a, b) => a.equals(b)) }, undefined);
+    }
+    setVisibleLines(visibleLines, stabilized) {
+        const visibleLineRanges = ( visibleLines.map((line) => ( new LineRange(line.startLineNumber, line.endLineNumber + 1))));
+        const state = ( new AttachedViewState(visibleLineRanges, stabilized));
+        this._state.set(state, undefined, undefined);
+        this.handleStateChange(state);
+    }
+}
+class AttachedViewHandler extends Disposable {
+    get lineRanges() { return this._lineRanges; }
+    constructor(_refreshTokens) {
+        super();
+        this._refreshTokens = _refreshTokens;
+        this.runner = this._register(( new RunOnceScheduler(() => this.update(), 50)));
+        this._computedLineRanges = [];
+        this._lineRanges = [];
+    }
+    update() {
+        if (equals(this._computedLineRanges, this._lineRanges, (a, b) => a.equals(b))) {
+            return;
+        }
+        this._computedLineRanges = this._lineRanges;
+        this._refreshTokens();
+    }
+    handleStateChange(state) {
+        this._lineRanges = state.visibleLineRanges;
+        if (state.stabilized) {
+            this.runner.cancel();
+            this.update();
+        }
+        else {
+            this.runner.schedule();
+        }
+    }
+}
+class AbstractSyntaxTokenBackend extends Disposable {
+    get backgroundTokenizationState() {
+        return this._backgroundTokenizationState;
+    }
+    constructor(_languageIdCodec, _textModel) {
+        super();
+        this._languageIdCodec = _languageIdCodec;
+        this._textModel = _textModel;
+        this._onDidChangeTokens = this._register(( new Emitter()));
+        this.onDidChangeTokens = this._onDidChangeTokens.event;
+    }
+    tokenizeIfCheap(lineNumber) {
+        if (this.isCheapToTokenize(lineNumber)) {
+            this.forceTokenization(lineNumber);
+        }
+    }
+}
+
+export { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViewState, AttachedViews };

package/vscode/src/vs/editor/common/model/{tokenizationTextModelPart.d.ts → tokens/tokenizationTextModelPart.d.ts}

@@ -1,20 +1,22 @@
-import { Event } from "../../../base/common/event.js";
-import { IPosition } from "../core/position.js";
-import { Range } from "../core/range.js";
-import { IWordAtPosition } from "../core/wordHelper.js";
-import { StandardTokenType } from "../encodedTokenAttributes.js";
-import { ILanguageService } from "../languages/language.service.js";
-import { LanguageConfigurationServiceChangeEvent } from "../languages/languageConfigurationRegistry.js";
-import { ILanguageConfigurationService } from "../languages/languageConfigurationRegistry.service.js";
-import { BracketPairsTextModelPart } from "./bracketPairsTextModelPart/bracketPairsImpl.js";
-import { TextModel } from "./textModel.js";
-import { TextModelPart } from "./textModelPart.js";
-import { AttachedViews } from "./tokens.js";
-import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from "../textModelEvents.js";
-import { BackgroundTokenizationState, ITokenizationTextModelPart } from "../tokenizationTextModelPart.js";
-import { LineTokens } from "../tokens/lineTokens.js";
-import { SparseMultilineTokens } from "../tokens/sparseMultilineTokens.js";
-import { IInstantiationService } from "../../../platform/instantiation/common/instantiation.js";
+import { Event } from "../../../../base/common/event.js";
+import { IPosition } from "../../core/position.js";
+import { Range } from "../../core/range.js";
+import { IWordAtPosition } from "../../core/wordHelper.js";
+import { StandardTokenType } from "../../encodedTokenAttributes.js";
+import { ILanguageService } from "../../languages/language.service.js";
+import { LanguageConfigurationServiceChangeEvent } from "../../languages/languageConfigurationRegistry.js";
+import { ILanguageConfigurationService } from "../../languages/languageConfigurationRegistry.service.js";
+import { BracketPairsTextModelPart } from "../bracketPairsTextModelPart/bracketPairsImpl.js";
+import { TextModel } from "../textModel.js";
+import { TextModelPart } from "../textModelPart.js";
+import { AbstractSyntaxTokenBackend, AttachedViews } from "./abstractSyntaxTokenBackend.js";
+import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from "../../textModelEvents.js";
+import { ITokenizationTextModelPart } from "../../tokenizationTextModelPart.js";
+import { LineTokens } from "../../tokens/lineTokens.js";
+import { SparseMultilineTokens } from "../../tokens/sparseMultilineTokens.js";
+import { IInstantiationService } from "../../../../platform/instantiation/common/instantiation.js";
+import { ITreeSitterLibraryService } from "../../services/treeSitter/treeSitterLibraryService.service.js";
+import { IObservable } from "../../../../base/common/observable.js";
 export declare class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {
     private readonly _textModel;
     private readonly _bracketPairsTextModelPart;
@@ -23,6 +25,7 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     private readonly _languageService;
     private readonly _languageConfigurationService;
     private readonly _instantiationService;
+    private readonly _treeSitterLibraryService;
     private readonly _semanticTokens;
     private readonly _onDidChangeLanguage;
     readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;
@@ -30,13 +33,10 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;
     private readonly _onDidChangeTokens;
     readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;
-    private _tokens;
-    private readonly _tokensDisposables;
-    constructor(_textModel: TextModel, _bracketPairsTextModelPart: BracketPairsTextModelPart, _languageId: string, _attachedViews: AttachedViews, _languageService: ILanguageService, _languageConfigurationService: ILanguageConfigurationService, _instantiationService: IInstantiationService);
-    private createGrammarTokens;
-    private createTreeSitterTokens;
-    private createTokens;
-    private createPreferredTokenProvider;
+    readonly tokens: IObservable<AbstractSyntaxTokenBackend>;
+    private readonly _useTreeSitter;
+    private readonly _languageIdObs;
+    constructor(_textModel: TextModel, _bracketPairsTextModelPart: BracketPairsTextModelPart, _languageId: string, _attachedViews: AttachedViews, _languageService: ILanguageService, _languageConfigurationService: ILanguageConfigurationService, _instantiationService: IInstantiationService, _treeSitterLibraryService: ITreeSitterLibraryService);
     _hasListeners(): boolean;
     handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void;
     handleDidChangeContent(e: IModelContentChangedEvent): void;
@@ -46,7 +46,7 @@ export declare class TokenizationTextModelPart extends TextModelPart implements
     private validateLineNumber;
     get hasTokens(): boolean;
     resetTokenization(): void;
-    get backgroundTokenizationState(): BackgroundTokenizationState;
+    get backgroundTokenizationState(): import("../../tokenizationTextModelPart.js").BackgroundTokenizationState;
     forceTokenization(lineNumber: number): void;
     hasAccurateTokensForLine(lineNumber: number): boolean;
     isCheapToTokenize(lineNumber: number): boolean;

package/vscode/src/vs/editor/common/model/tokens/tokenizationTextModelPart.js

@@ -0,0 +1,255 @@
+
+import { __decorate, __param } from '../../../../../../../external/tslib/tslib.es6.js';
+import { CharCode } from '../../../../base/common/charCode.js';
+import { BugIndicatingError } from '../../../../base/common/errors.js';
+import { Emitter } from '../../../../base/common/event.js';
+import { countEOL } from '../../core/misc/eolCounter.js';
+import { Position } from '../../core/position.js';
+import { getWordAtText } from '../../core/wordHelper.js';
+import { ILanguageService } from '../../languages/language.service.js';
+import { ILanguageConfigurationService } from '../../languages/languageConfigurationRegistry.service.js';
+import { TextModelPart } from '../textModelPart.js';
+import { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';
+import { SparseTokensStore } from '../../tokens/sparseTokensStore.js';
+import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
+import { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';
+import { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.service.js';
+import '../../../../base/common/observableInternal/index.js';
+import { observableValue } from '../../../../base/common/observableInternal/observables/observableValue.js';
+import { derived } from '../../../../base/common/observableInternal/observables/derived.js';
+
+var TokenizationTextModelPart_1;
+let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
+    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _instantiationService, _treeSitterLibraryService) {
+        super();
+        this._textModel = _textModel;
+        this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
+        this._languageId = _languageId;
+        this._attachedViews = _attachedViews;
+        this._languageService = _languageService;
+        this._languageConfigurationService = _languageConfigurationService;
+        this._instantiationService = _instantiationService;
+        this._treeSitterLibraryService = _treeSitterLibraryService;
+        this._languageIdObs = observableValue(this, this._languageId);
+        this._useTreeSitter = derived(this, reader => {
+            const languageId = this._languageIdObs.read(reader);
+            return this._treeSitterLibraryService.supportsLanguage(languageId, reader);
+        });
+        this.tokens = derived(this, reader => {
+            let tokens;
+            if (this._useTreeSitter.read(reader)) {
+                tokens = reader.store.add(this._instantiationService.createInstance(TreeSitterSyntaxTokenBackend, this._languageIdObs, this._languageService.languageIdCodec, this._textModel, this._attachedViews.visibleLineRanges));
+            }
+            else {
+                tokens = reader.store.add(( new TokenizerSyntaxTokenBackend(
+                    this._languageService.languageIdCodec,
+                    this._textModel,
+                    () => this._languageId,
+                    this._attachedViews
+                )));
+            }
+            reader.store.add(tokens.onDidChangeTokens(e => {
+                this._emitModelTokensChangedEvent(e);
+            }));
+            reader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {
+                this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
+            }));
+            return tokens;
+        });
+        let hadTokens = false;
+        this.tokens.recomputeInitiallyAndOnChange(this._store, value => {
+            if (hadTokens) {
+                value.todo_resetTokenization();
+            }
+            hadTokens = true;
+        });
+        this._semanticTokens = ( new SparseTokensStore(this._languageService.languageIdCodec));
+        this._onDidChangeLanguage = this._register(( new Emitter()));
+        this.onDidChangeLanguage = this._onDidChangeLanguage.event;
+        this._onDidChangeLanguageConfiguration = this._register(( new Emitter()));
+        this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
+        this._onDidChangeTokens = this._register(( new Emitter()));
+        this.onDidChangeTokens = this._onDidChangeTokens.event;
+    }
+    _hasListeners() {
+        return (this._onDidChangeLanguage.hasListeners()
+            || this._onDidChangeLanguageConfiguration.hasListeners()
+            || this._onDidChangeTokens.hasListeners());
+    }
+    handleLanguageConfigurationServiceChange(e) {
+        if (e.affects(this._languageId)) {
+            this._onDidChangeLanguageConfiguration.fire({});
+        }
+    }
+    handleDidChangeContent(e) {
+        if (e.isFlush) {
+            this._semanticTokens.flush();
+        }
+        else if (!e.isEolChange) {
+            for (const c of e.changes) {
+                const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
+                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : CharCode.Null);
+            }
+        }
+        this.tokens.get().handleDidChangeContent(e);
+    }
+    handleDidChangeAttached() {
+        this.tokens.get().handleDidChangeAttached();
+    }
+    getLineTokens(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        const syntacticTokens = this.tokens.get().getLineTokens(lineNumber);
+        return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
+    }
+    _emitModelTokensChangedEvent(e) {
+        if (!this._textModel._isDisposing()) {
+            this._bracketPairsTextModelPart.handleDidChangeTokens(e);
+            this._onDidChangeTokens.fire(e);
+        }
+    }
+    validateLineNumber(lineNumber) {
+        if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
+            throw ( new BugIndicatingError('Illegal value for lineNumber'));
+        }
+    }
+    get hasTokens() {
+        return this.tokens.get().hasTokens;
+    }
+    resetTokenization() {
+        this.tokens.get().todo_resetTokenization();
+    }
+    get backgroundTokenizationState() {
+        return this.tokens.get().backgroundTokenizationState;
+    }
+    forceTokenization(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        this.tokens.get().forceTokenization(lineNumber);
+    }
+    hasAccurateTokensForLine(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        return this.tokens.get().hasAccurateTokensForLine(lineNumber);
+    }
+    isCheapToTokenize(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        return this.tokens.get().isCheapToTokenize(lineNumber);
+    }
+    tokenizeIfCheap(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        this.tokens.get().tokenizeIfCheap(lineNumber);
+    }
+    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
+        return this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);
+    }
+    tokenizeLinesAt(lineNumber, lines) {
+        return this.tokens.get().tokenizeLinesAt(lineNumber, lines);
+    }
+    setSemanticTokens(tokens, isComplete) {
+        this._semanticTokens.set(tokens, isComplete, this._textModel);
+        this._emitModelTokensChangedEvent({
+            semanticTokensApplied: tokens !== null,
+            ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],
+        });
+    }
+    hasCompleteSemanticTokens() {
+        return this._semanticTokens.isComplete();
+    }
+    hasSomeSemanticTokens() {
+        return !this._semanticTokens.isEmpty();
+    }
+    setPartialSemanticTokens(range, tokens) {
+        if (this.hasCompleteSemanticTokens()) {
+            return;
+        }
+        const changedRange = this._textModel.validateRange(this._semanticTokens.setPartial(range, tokens));
+        this._emitModelTokensChangedEvent({
+            semanticTokensApplied: true,
+            ranges: [
+                {
+                    fromLineNumber: changedRange.startLineNumber,
+                    toLineNumber: changedRange.endLineNumber,
+                },
+            ],
+        });
+    }
+    getWordAtPosition(_position) {
+        this.assertNotDisposed();
+        const position = this._textModel.validatePosition(_position);
+        const lineContent = this._textModel.getLineContent(position.lineNumber);
+        const lineTokens = this.getLineTokens(position.lineNumber);
+        const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
+        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex);
+        const rightBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), lineContent.substring(rbStartOffset, rbEndOffset), rbStartOffset);
+        if (rightBiasedWord &&
+            rightBiasedWord.startColumn <= _position.column &&
+            _position.column <= rightBiasedWord.endColumn) {
+            return rightBiasedWord;
+        }
+        if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
+            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex - 1);
+            const leftBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), lineContent.substring(lbStartOffset, lbEndOffset), lbStartOffset);
+            if (leftBiasedWord &&
+                leftBiasedWord.startColumn <= _position.column &&
+                _position.column <= leftBiasedWord.endColumn) {
+                return leftBiasedWord;
+            }
+        }
+        return null;
+    }
+    getLanguageConfiguration(languageId) {
+        return this._languageConfigurationService.getLanguageConfiguration(languageId);
+    }
+    static _findLanguageBoundaries(lineTokens, tokenIndex) {
+        const languageId = lineTokens.getLanguageId(tokenIndex);
+        let startOffset = 0;
+        for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {
+            startOffset = lineTokens.getStartOffset(i);
+        }
+        let endOffset = lineTokens.getLineContent().length;
+        for (let i = tokenIndex, tokenCount = lineTokens.getCount(); i < tokenCount && lineTokens.getLanguageId(i) === languageId; i++) {
+            endOffset = lineTokens.getEndOffset(i);
+        }
+        return [startOffset, endOffset];
+    }
+    getWordUntilPosition(position) {
+        const wordAtPosition = this.getWordAtPosition(position);
+        if (!wordAtPosition) {
+            return { word: '', startColumn: position.column, endColumn: position.column, };
+        }
+        return {
+            word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
+            startColumn: wordAtPosition.startColumn,
+            endColumn: position.column,
+        };
+    }
+    getLanguageId() {
+        return this._languageId;
+    }
+    getLanguageIdAtPosition(lineNumber, column) {
+        const position = this._textModel.validatePosition(( new Position(lineNumber, column)));
+        const lineTokens = this.getLineTokens(position.lineNumber);
+        return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
+    }
+    setLanguageId(languageId, source = 'api') {
+        if (this._languageId === languageId) {
+            return;
+        }
+        const e = {
+            oldLanguage: this._languageId,
+            newLanguage: languageId,
+            source
+        };
+        this._languageId = languageId;
+        this._languageIdObs.set(languageId, undefined);
+        this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
+        this._onDidChangeLanguage.fire(e);
+        this._onDidChangeLanguageConfiguration.fire({});
+    }
+};
+TokenizationTextModelPart = TokenizationTextModelPart_1 = ( __decorate([
+    ( __param(4, ILanguageService)),
+    ( __param(5, ILanguageConfigurationService)),
+    ( __param(6, IInstantiationService)),
+    ( __param(7, ITreeSitterLibraryService))
+], TokenizationTextModelPart));
+
+export { TokenizationTextModelPart };

package/vscode/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.d.ts

@@ -0,0 +1,37 @@
+import { Emitter, Event } from "../../../../base/common/event.js";
+import { StandardTokenType } from "../../encodedTokenAttributes.js";
+import { ILanguageIdCodec } from "../../languages.js";
+import { IModelContentChangedEvent } from "../../textModelEvents.js";
+import { BackgroundTokenizationState } from "../../tokenizationTextModelPart.js";
+import { LineTokens } from "../../tokens/lineTokens.js";
+import { TextModel } from "../textModel.js";
+import { AbstractSyntaxTokenBackend, AttachedViews } from "./abstractSyntaxTokenBackend.js";
+export declare class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
+    private readonly getLanguageId;
+    private _tokenizer;
+    protected _backgroundTokenizationState: BackgroundTokenizationState;
+    protected readonly _onDidChangeBackgroundTokenizationState: Emitter<void>;
+    readonly onDidChangeBackgroundTokenizationState: Event<void>;
+    private _defaultBackgroundTokenizer;
+    private readonly _backgroundTokenizer;
+    private readonly _tokens;
+    private _debugBackgroundTokens;
+    private _debugBackgroundStates;
+    private readonly _debugBackgroundTokenizer;
+    private readonly _attachedViewStates;
+    constructor(languageIdCodec: ILanguageIdCodec, textModel: TextModel, getLanguageId: () => string, attachedViews: AttachedViews);
+    todo_resetTokenization(fireTokenChangeEvent?: boolean): void;
+    handleDidChangeAttached(): void;
+    handleDidChangeContent(e: IModelContentChangedEvent): void;
+    private setTokens;
+    private refreshAllVisibleLineTokens;
+    private refreshRanges;
+    private refreshRange;
+    forceTokenization(lineNumber: number): void;
+    hasAccurateTokensForLine(lineNumber: number): boolean;
+    isCheapToTokenize(lineNumber: number): boolean;
+    getLineTokens(lineNumber: number): LineTokens;
+    getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
+    tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
+    get hasTokens(): boolean;
+}