monaco-editor11 1.0.9 → 1.1.0

package/dist/workers/editor/common/tokenization.js

@@ -1,287 +0,0 @@
-import { Color } from '../../../../base/common/color.js';
-
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-class ParsedTokenThemeRule {
-    constructor(token, index, fontStyle, foreground, background) {
-        this._parsedThemeRuleBrand = undefined;
-        this.token = token;
-        this.index = index;
-        this.fontStyle = fontStyle;
-        this.foreground = foreground;
-        this.background = background;
-    }
-}
-/**
- * Parse a raw theme into rules.
- */
-function parseTokenTheme(source) {
-    if (!source || !Array.isArray(source)) {
-        return [];
-    }
-    const result = [];
-    let resultLen = 0;
-    for (let i = 0, len = source.length; i < len; i++) {
-        const entry = source[i];
-        let fontStyle = -1 /* FontStyle.NotSet */;
-        if (typeof entry.fontStyle === 'string') {
-            fontStyle = 0 /* FontStyle.None */;
-            const segments = entry.fontStyle.split(' ');
-            for (let j = 0, lenJ = segments.length; j < lenJ; j++) {
-                const segment = segments[j];
-                switch (segment) {
-                    case 'italic':
-                        fontStyle = fontStyle | 1 /* FontStyle.Italic */;
-                        break;
-                    case 'bold':
-                        fontStyle = fontStyle | 2 /* FontStyle.Bold */;
-                        break;
-                    case 'underline':
-                        fontStyle = fontStyle | 4 /* FontStyle.Underline */;
-                        break;
-                    case 'strikethrough':
-                        fontStyle = fontStyle | 8 /* FontStyle.Strikethrough */;
-                        break;
-                }
-            }
-        }
-        let foreground = null;
-        if (typeof entry.foreground === 'string') {
-            foreground = entry.foreground;
-        }
-        let background = null;
-        if (typeof entry.background === 'string') {
-            background = entry.background;
-        }
-        result[resultLen++] = new ParsedTokenThemeRule(entry.token || '', i, fontStyle, foreground, background);
-    }
-    return result;
-}
-/**
- * Resolve rules (i.e. inheritance).
- */
-function resolveParsedTokenThemeRules(parsedThemeRules, customTokenColors) {
-    // Sort rules lexicographically, and then by index if necessary
-    parsedThemeRules.sort((a, b) => {
-        const r = strcmp(a.token, b.token);
-        if (r !== 0) {
-            return r;
-        }
-        return a.index - b.index;
-    });
-    // Determine defaults
-    let defaultFontStyle = 0 /* FontStyle.None */;
-    let defaultForeground = '000000';
-    let defaultBackground = 'ffffff';
-    while (parsedThemeRules.length >= 1 && parsedThemeRules[0].token === '') {
-        const incomingDefaults = parsedThemeRules.shift();
-        if (incomingDefaults.fontStyle !== -1 /* FontStyle.NotSet */) {
-            defaultFontStyle = incomingDefaults.fontStyle;
-        }
-        if (incomingDefaults.foreground !== null) {
-            defaultForeground = incomingDefaults.foreground;
-        }
-        if (incomingDefaults.background !== null) {
-            defaultBackground = incomingDefaults.background;
-        }
-    }
-    const colorMap = new ColorMap();
-    // start with token colors from custom token themes
-    for (const color of customTokenColors) {
-        colorMap.getId(color);
-    }
-    const foregroundColorId = colorMap.getId(defaultForeground);
-    const backgroundColorId = colorMap.getId(defaultBackground);
-    const defaults = new ThemeTrieElementRule(defaultFontStyle, foregroundColorId, backgroundColorId);
-    const root = new ThemeTrieElement(defaults);
-    for (let i = 0, len = parsedThemeRules.length; i < len; i++) {
-        const rule = parsedThemeRules[i];
-        root.insert(rule.token, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background));
-    }
-    return new TokenTheme(colorMap, root);
-}
-const colorRegExp = /^#?([0-9A-Fa-f]{6})([0-9A-Fa-f]{2})?$/;
-class ColorMap {
-    constructor() {
-        this._lastColorId = 0;
-        this._id2color = [];
-        this._color2id = new Map();
-    }
-    getId(color) {
-        if (color === null) {
-            return 0;
-        }
-        const match = color.match(colorRegExp);
-        if (!match) {
-            throw new Error('Illegal value for token color: ' + color);
-        }
-        color = match[1].toUpperCase();
-        let value = this._color2id.get(color);
-        if (value) {
-            return value;
-        }
-        value = ++this._lastColorId;
-        this._color2id.set(color, value);
-        this._id2color[value] = Color.fromHex('#' + color);
-        return value;
-    }
-    getColorMap() {
-        return this._id2color.slice(0);
-    }
-}
-class TokenTheme {
-    static createFromRawTokenTheme(source, customTokenColors) {
-        return this.createFromParsedTokenTheme(parseTokenTheme(source), customTokenColors);
-    }
-    static createFromParsedTokenTheme(source, customTokenColors) {
-        return resolveParsedTokenThemeRules(source, customTokenColors);
-    }
-    constructor(colorMap, root) {
-        this._colorMap = colorMap;
-        this._root = root;
-        this._cache = new Map();
-    }
-    getColorMap() {
-        return this._colorMap.getColorMap();
-    }
-    _match(token) {
-        return this._root.match(token);
-    }
-    match(languageId, token) {
-        // The cache contains the metadata without the language bits set.
-        let result = this._cache.get(token);
-        if (typeof result === 'undefined') {
-            const rule = this._match(token);
-            const standardToken = toStandardTokenType(token);
-            result = (rule.metadata
-                | (standardToken << 8 /* MetadataConsts.TOKEN_TYPE_OFFSET */)) >>> 0;
-            this._cache.set(token, result);
-        }
-        return (result
-            | (languageId << 0 /* MetadataConsts.LANGUAGEID_OFFSET */)) >>> 0;
-    }
-}
-const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|regexp)\b/;
-function toStandardTokenType(tokenType) {
-    const m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
-    if (!m) {
-        return 0 /* StandardTokenType.Other */;
-    }
-    switch (m[1]) {
-        case 'comment':
-            return 1 /* StandardTokenType.Comment */;
-        case 'string':
-            return 2 /* StandardTokenType.String */;
-        case 'regex':
-            return 3 /* StandardTokenType.RegEx */;
-        case 'regexp':
-            return 3 /* StandardTokenType.RegEx */;
-    }
-    throw new Error('Unexpected match for standard token type!');
-}
-function strcmp(a, b) {
-    if (a < b) {
-        return -1;
-    }
-    if (a > b) {
-        return 1;
-    }
-    return 0;
-}
-class ThemeTrieElementRule {
-    constructor(fontStyle, foreground, background) {
-        this._themeTrieElementRuleBrand = undefined;
-        this._fontStyle = fontStyle;
-        this._foreground = foreground;
-        this._background = background;
-        this.metadata = ((this._fontStyle << 11 /* MetadataConsts.FONT_STYLE_OFFSET */)
-            | (this._foreground << 15 /* MetadataConsts.FOREGROUND_OFFSET */)
-            | (this._background << 24 /* MetadataConsts.BACKGROUND_OFFSET */)) >>> 0;
-    }
-    clone() {
-        return new ThemeTrieElementRule(this._fontStyle, this._foreground, this._background);
-    }
-    acceptOverwrite(fontStyle, foreground, background) {
-        if (fontStyle !== -1 /* FontStyle.NotSet */) {
-            this._fontStyle = fontStyle;
-        }
-        if (foreground !== 0 /* ColorId.None */) {
-            this._foreground = foreground;
-        }
-        if (background !== 0 /* ColorId.None */) {
-            this._background = background;
-        }
-        this.metadata = ((this._fontStyle << 11 /* MetadataConsts.FONT_STYLE_OFFSET */)
-            | (this._foreground << 15 /* MetadataConsts.FOREGROUND_OFFSET */)
-            | (this._background << 24 /* MetadataConsts.BACKGROUND_OFFSET */)) >>> 0;
-    }
-}
-class ThemeTrieElement {
-    constructor(mainRule) {
-        this._themeTrieElementBrand = undefined;
-        this._mainRule = mainRule;
-        this._children = new Map();
-    }
-    match(token) {
-        if (token === '') {
-            return this._mainRule;
-        }
-        const dotIndex = token.indexOf('.');
-        let head;
-        let tail;
-        if (dotIndex === -1) {
-            head = token;
-            tail = '';
-        }
-        else {
-            head = token.substring(0, dotIndex);
-            tail = token.substring(dotIndex + 1);
-        }
-        const child = this._children.get(head);
-        if (typeof child !== 'undefined') {
-            return child.match(tail);
-        }
-        return this._mainRule;
-    }
-    insert(token, fontStyle, foreground, background) {
-        if (token === '') {
-            // Merge into the main rule
-            this._mainRule.acceptOverwrite(fontStyle, foreground, background);
-            return;
-        }
-        const dotIndex = token.indexOf('.');
-        let head;
-        let tail;
-        if (dotIndex === -1) {
-            head = token;
-            tail = '';
-        }
-        else {
-            head = token.substring(0, dotIndex);
-            tail = token.substring(dotIndex + 1);
-        }
-        let child = this._children.get(head);
-        if (typeof child === 'undefined') {
-            child = new ThemeTrieElement(this._mainRule.clone());
-            this._children.set(head, child);
-        }
-        child.insert(tail, fontStyle, foreground, background);
-    }
-}
-function generateTokensCSSForColorMap(colorMap) {
-    const rules = [];
-    for (let i = 1, len = colorMap.length; i < len; i++) {
-        const color = colorMap[i];
-        rules[i] = `.mtk${i} { color: ${color}; }`;
-    }
-    rules.push('.mtki { font-style: italic; }');
-    rules.push('.mtkb { font-weight: bold; }');
-    rules.push('.mtku { text-decoration: underline; text-underline-position: under; }');
-    rules.push('.mtks { text-decoration: line-through; }');
-    rules.push('.mtks.mtku { text-decoration: underline line-through; text-underline-position: under; }');
-    return rules.join('\n');
-}
-
-export { ColorMap, ParsedTokenThemeRule, ThemeTrieElement, ThemeTrieElementRule, TokenTheme, generateTokensCSSForColorMap, parseTokenTheme, strcmp, toStandardTokenType };

package/dist/workers/editor/common/tokenizationRegistry.js

@@ -1,123 +0,0 @@
-import { Emitter } from '../../base/common/event.js';
-import { toDisposable, Disposable } from '../../base/common/lifecycle.js';
-
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-class TokenizationRegistry {
-    constructor() {
-        this._tokenizationSupports = new Map();
-        this._factories = new Map();
-        this._onDidChange = new Emitter();
-        this.onDidChange = this._onDidChange.event;
-        this._colorMap = null;
-    }
-    handleChange(languageIds) {
-        this._onDidChange.fire({
-            changedLanguages: languageIds,
-            changedColorMap: false
-        });
-    }
-    register(languageId, support) {
-        this._tokenizationSupports.set(languageId, support);
-        this.handleChange([languageId]);
-        return toDisposable(() => {
-            if (this._tokenizationSupports.get(languageId) !== support) {
-                return;
-            }
-            this._tokenizationSupports.delete(languageId);
-            this.handleChange([languageId]);
-        });
-    }
-    get(languageId) {
-        return this._tokenizationSupports.get(languageId) || null;
-    }
-    registerFactory(languageId, factory) {
-        this._factories.get(languageId)?.dispose();
-        const myData = new TokenizationSupportFactoryData(this, languageId, factory);
-        this._factories.set(languageId, myData);
-        return toDisposable(() => {
-            const v = this._factories.get(languageId);
-            if (!v || v !== myData) {
-                return;
-            }
-            this._factories.delete(languageId);
-            v.dispose();
-        });
-    }
-    async getOrCreate(languageId) {
-        // check first if the support is already set
-        const tokenizationSupport = this.get(languageId);
-        if (tokenizationSupport) {
-            return tokenizationSupport;
-        }
-        const factory = this._factories.get(languageId);
-        if (!factory || factory.isResolved) {
-            // no factory or factory.resolve already finished
-            return null;
-        }
-        await factory.resolve();
-        return this.get(languageId);
-    }
-    isResolved(languageId) {
-        const tokenizationSupport = this.get(languageId);
-        if (tokenizationSupport) {
-            return true;
-        }
-        const factory = this._factories.get(languageId);
-        if (!factory || factory.isResolved) {
-            return true;
-        }
-        return false;
-    }
-    setColorMap(colorMap) {
-        this._colorMap = colorMap;
-        this._onDidChange.fire({
-            changedLanguages: Array.from(this._tokenizationSupports.keys()),
-            changedColorMap: true
-        });
-    }
-    getColorMap() {
-        return this._colorMap;
-    }
-    getDefaultBackground() {
-        if (this._colorMap && this._colorMap.length > 2 /* ColorId.DefaultBackground */) {
-            return this._colorMap[2 /* ColorId.DefaultBackground */];
-        }
-        return null;
-    }
-}
-class TokenizationSupportFactoryData extends Disposable {
-    get isResolved() {
-        return this._isResolved;
-    }
-    constructor(_registry, _languageId, _factory) {
-        super();
-        this._registry = _registry;
-        this._languageId = _languageId;
-        this._factory = _factory;
-        this._isDisposed = false;
-        this._resolvePromise = null;
-        this._isResolved = false;
-    }
-    dispose() {
-        this._isDisposed = true;
-        super.dispose();
-    }
-    async resolve() {
-        if (!this._resolvePromise) {
-            this._resolvePromise = this._create();
-        }
-        return this._resolvePromise;
-    }
-    async _create() {
-        const value = await this._factory.tokenizationSupport;
-        this._isResolved = true;
-        if (value && !this._isDisposed) {
-            this._register(this._registry.register(this._languageId, value));
-        }
-    }
-}
-
-export { TokenizationRegistry };

package/dist/workers/editor/common/tokenizationTextModelPart.js

@@ -1,275 +0,0 @@
-import { BugIndicatingError } from '../../../../base/common/errors.js';
-import { Emitter } from '../../../../base/common/event.js';
-import { countEOL } from '../../core/misc/eolCounter.js';
-import { Position } from '../../core/position.js';
-import { getWordAtText } from '../../core/wordHelper.js';
-import { ILanguageService } from '../../languages/language.js';
-import { ILanguageConfigurationService } from '../../languages/languageConfigurationRegistry.js';
-import { TextModelPart } from '../textModelPart.js';
-import { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';
-import { SparseTokensStore } from '../../tokens/sparseTokensStore.js';
-import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
-import { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';
-import { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';
-import '../../../../base/common/observableInternal/index.js';
-import { observableValue } from '../../../../base/common/observableInternal/observables/observableValue.js';
-import { derived } from '../../../../base/common/observableInternal/observables/derived.js';
-
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-var __decorate = (undefined && undefined.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var __param = (undefined && undefined.__param) || function (paramIndex, decorator) {
-    return function (target, key) { decorator(target, key, paramIndex); }
-};
-var TokenizationTextModelPart_1;
-let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
-    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _instantiationService, _treeSitterLibraryService) {
-        super();
-        this._textModel = _textModel;
-        this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
-        this._languageId = _languageId;
-        this._attachedViews = _attachedViews;
-        this._languageService = _languageService;
-        this._languageConfigurationService = _languageConfigurationService;
-        this._instantiationService = _instantiationService;
-        this._treeSitterLibraryService = _treeSitterLibraryService;
-        this._languageIdObs = observableValue(this, this._languageId);
-        this._useTreeSitter = derived(this, reader => {
-            const languageId = this._languageIdObs.read(reader);
-            return this._treeSitterLibraryService.supportsLanguage(languageId, reader);
-        });
-        this.tokens = derived(this, reader => {
-            let tokens;
-            if (this._useTreeSitter.read(reader)) {
-                tokens = reader.store.add(this._instantiationService.createInstance(TreeSitterSyntaxTokenBackend, this._languageIdObs, this._languageService.languageIdCodec, this._textModel, this._attachedViews.visibleLineRanges));
-            }
-            else {
-                tokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));
-            }
-            reader.store.add(tokens.onDidChangeTokens(e => {
-                this._emitModelTokensChangedEvent(e);
-            }));
-            reader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {
-                this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
-            }));
-            return tokens;
-        });
-        let hadTokens = false;
-        this.tokens.recomputeInitiallyAndOnChange(this._store, value => {
-            if (hadTokens) {
-                // We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.
-                // TODO@hediet: Look into why this is needed.
-                value.todo_resetTokenization();
-            }
-            hadTokens = true;
-        });
-        this._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);
-        this._onDidChangeLanguage = this._register(new Emitter());
-        this.onDidChangeLanguage = this._onDidChangeLanguage.event;
-        this._onDidChangeLanguageConfiguration = this._register(new Emitter());
-        this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
-        this._onDidChangeTokens = this._register(new Emitter());
-        this.onDidChangeTokens = this._onDidChangeTokens.event;
-    }
-    handleLanguageConfigurationServiceChange(e) {
-        if (e.affects(this._languageId)) {
-            this._onDidChangeLanguageConfiguration.fire({});
-        }
-    }
-    handleDidChangeContent(e) {
-        if (e.isFlush) {
-            this._semanticTokens.flush();
-        }
-        else if (!e.isEolChange) { // We don't have to do anything on an EOL change
-            for (const c of e.changes) {
-                const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
-                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : 0 /* CharCode.Null */);
-            }
-        }
-        this.tokens.get().handleDidChangeContent(e);
-    }
-    handleDidChangeAttached() {
-        this.tokens.get().handleDidChangeAttached();
-    }
-    /**
-     * Includes grammar and semantic tokens.
-     */
-    getLineTokens(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        const syntacticTokens = this.tokens.get().getLineTokens(lineNumber);
-        return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
-    }
-    _emitModelTokensChangedEvent(e) {
-        if (!this._textModel._isDisposing()) {
-            this._bracketPairsTextModelPart.handleDidChangeTokens(e);
-            this._onDidChangeTokens.fire(e);
-        }
-    }
-    // #region Grammar Tokens
-    validateLineNumber(lineNumber) {
-        if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
-            throw new BugIndicatingError('Illegal value for lineNumber');
-        }
-    }
-    get hasTokens() {
-        return this.tokens.get().hasTokens;
-    }
-    resetTokenization() {
-        this.tokens.get().todo_resetTokenization();
-    }
-    get backgroundTokenizationState() {
-        return this.tokens.get().backgroundTokenizationState;
-    }
-    forceTokenization(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        this.tokens.get().forceTokenization(lineNumber);
-    }
-    hasAccurateTokensForLine(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        return this.tokens.get().hasAccurateTokensForLine(lineNumber);
-    }
-    isCheapToTokenize(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        return this.tokens.get().isCheapToTokenize(lineNumber);
-    }
-    tokenizeIfCheap(lineNumber) {
-        this.validateLineNumber(lineNumber);
-        this.tokens.get().tokenizeIfCheap(lineNumber);
-    }
-    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
-        return this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);
-    }
-    tokenizeLinesAt(lineNumber, lines) {
-        return this.tokens.get().tokenizeLinesAt(lineNumber, lines);
-    }
-    // #endregion
-    // #region Semantic Tokens
-    setSemanticTokens(tokens, isComplete) {
-        this._semanticTokens.set(tokens, isComplete, this._textModel);
-        this._emitModelTokensChangedEvent({
-            semanticTokensApplied: tokens !== null,
-            ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],
-        });
-    }
-    hasCompleteSemanticTokens() {
-        return this._semanticTokens.isComplete();
-    }
-    hasSomeSemanticTokens() {
-        return !this._semanticTokens.isEmpty();
-    }
-    setPartialSemanticTokens(range, tokens) {
-        if (this.hasCompleteSemanticTokens()) {
-            return;
-        }
-        const changedRange = this._textModel.validateRange(this._semanticTokens.setPartial(range, tokens));
-        this._emitModelTokensChangedEvent({
-            semanticTokensApplied: true,
-            ranges: [
-                {
-                    fromLineNumber: changedRange.startLineNumber,
-                    toLineNumber: changedRange.endLineNumber,
-                },
-            ],
-        });
-    }
-    // #endregion
-    // #region Utility Methods
-    getWordAtPosition(_position) {
-        this.assertNotDisposed();
-        const position = this._textModel.validatePosition(_position);
-        const lineContent = this._textModel.getLineContent(position.lineNumber);
-        const lineTokens = this.getLineTokens(position.lineNumber);
-        const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
-        // (1). First try checking right biased word
-        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex);
-        const rightBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), lineContent.substring(rbStartOffset, rbEndOffset), rbStartOffset);
-        // Make sure the result touches the original passed in position
-        if (rightBiasedWord &&
-            rightBiasedWord.startColumn <= _position.column &&
-            _position.column <= rightBiasedWord.endColumn) {
-            return rightBiasedWord;
-        }
-        // (2). Else, if we were at a language boundary, check the left biased word
-        if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
-            // edge case, where `position` sits between two tokens belonging to two different languages
-            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex - 1);
-            const leftBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), lineContent.substring(lbStartOffset, lbEndOffset), lbStartOffset);
-            // Make sure the result touches the original passed in position
-            if (leftBiasedWord &&
-                leftBiasedWord.startColumn <= _position.column &&
-                _position.column <= leftBiasedWord.endColumn) {
-                return leftBiasedWord;
-            }
-        }
-        return null;
-    }
-    getLanguageConfiguration(languageId) {
-        return this._languageConfigurationService.getLanguageConfiguration(languageId);
-    }
-    static _findLanguageBoundaries(lineTokens, tokenIndex) {
-        const languageId = lineTokens.getLanguageId(tokenIndex);
-        // go left until a different language is hit
-        let startOffset = 0;
-        for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {
-            startOffset = lineTokens.getStartOffset(i);
-        }
-        // go right until a different language is hit
-        let endOffset = lineTokens.getLineContent().length;
-        for (let i = tokenIndex, tokenCount = lineTokens.getCount(); i < tokenCount && lineTokens.getLanguageId(i) === languageId; i++) {
-            endOffset = lineTokens.getEndOffset(i);
-        }
-        return [startOffset, endOffset];
-    }
-    getWordUntilPosition(position) {
-        const wordAtPosition = this.getWordAtPosition(position);
-        if (!wordAtPosition) {
-            return { word: '', startColumn: position.column, endColumn: position.column, };
-        }
-        return {
-            word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
-            startColumn: wordAtPosition.startColumn,
-            endColumn: position.column,
-        };
-    }
-    // #endregion
-    // #region Language Id handling
-    getLanguageId() {
-        return this._languageId;
-    }
-    getLanguageIdAtPosition(lineNumber, column) {
-        const position = this._textModel.validatePosition(new Position(lineNumber, column));
-        const lineTokens = this.getLineTokens(position.lineNumber);
-        return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
-    }
-    setLanguageId(languageId, source = 'api') {
-        if (this._languageId === languageId) {
-            // There's nothing to do
-            return;
-        }
-        const e = {
-            oldLanguage: this._languageId,
-            newLanguage: languageId,
-            source
-        };
-        this._languageId = languageId;
-        this._languageIdObs.set(languageId, undefined);
-        this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
-        this._onDidChangeLanguage.fire(e);
-        this._onDidChangeLanguageConfiguration.fire({});
-    }
-};
-TokenizationTextModelPart = TokenizationTextModelPart_1 = __decorate([
-    __param(4, ILanguageService),
-    __param(5, ILanguageConfigurationService),
-    __param(6, IInstantiationService),
-    __param(7, ITreeSitterLibraryService)
-], TokenizationTextModelPart);
-
-export { TokenizationTextModelPart };