monaco-editor11 1.0.7 → 1.0.9

package/dist/workers/editor/common/tokenization.js

@@ -0,0 +1,287 @@
+import { Color } from '../../../../base/common/color.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+class ParsedTokenThemeRule {
+    constructor(token, index, fontStyle, foreground, background) {
+        this._parsedThemeRuleBrand = undefined;
+        this.token = token;
+        this.index = index;
+        this.fontStyle = fontStyle;
+        this.foreground = foreground;
+        this.background = background;
+    }
+}
+/**
+ * Parse a raw theme into rules.
+ */
+function parseTokenTheme(source) {
+    if (!source || !Array.isArray(source)) {
+        return [];
+    }
+    const result = [];
+    let resultLen = 0;
+    for (let i = 0, len = source.length; i < len; i++) {
+        const entry = source[i];
+        let fontStyle = -1 /* FontStyle.NotSet */;
+        if (typeof entry.fontStyle === 'string') {
+            fontStyle = 0 /* FontStyle.None */;
+            const segments = entry.fontStyle.split(' ');
+            for (let j = 0, lenJ = segments.length; j < lenJ; j++) {
+                const segment = segments[j];
+                switch (segment) {
+                    case 'italic':
+                        fontStyle = fontStyle | 1 /* FontStyle.Italic */;
+                        break;
+                    case 'bold':
+                        fontStyle = fontStyle | 2 /* FontStyle.Bold */;
+                        break;
+                    case 'underline':
+                        fontStyle = fontStyle | 4 /* FontStyle.Underline */;
+                        break;
+                    case 'strikethrough':
+                        fontStyle = fontStyle | 8 /* FontStyle.Strikethrough */;
+                        break;
+                }
+            }
+        }
+        let foreground = null;
+        if (typeof entry.foreground === 'string') {
+            foreground = entry.foreground;
+        }
+        let background = null;
+        if (typeof entry.background === 'string') {
+            background = entry.background;
+        }
+        result[resultLen++] = new ParsedTokenThemeRule(entry.token || '', i, fontStyle, foreground, background);
+    }
+    return result;
+}
+/**
+ * Resolve rules (i.e. inheritance).
+ */
+function resolveParsedTokenThemeRules(parsedThemeRules, customTokenColors) {
+    // Sort rules lexicographically, and then by index if necessary
+    parsedThemeRules.sort((a, b) => {
+        const r = strcmp(a.token, b.token);
+        if (r !== 0) {
+            return r;
+        }
+        return a.index - b.index;
+    });
+    // Determine defaults
+    let defaultFontStyle = 0 /* FontStyle.None */;
+    let defaultForeground = '000000';
+    let defaultBackground = 'ffffff';
+    while (parsedThemeRules.length >= 1 && parsedThemeRules[0].token === '') {
+        const incomingDefaults = parsedThemeRules.shift();
+        if (incomingDefaults.fontStyle !== -1 /* FontStyle.NotSet */) {
+            defaultFontStyle = incomingDefaults.fontStyle;
+        }
+        if (incomingDefaults.foreground !== null) {
+            defaultForeground = incomingDefaults.foreground;
+        }
+        if (incomingDefaults.background !== null) {
+            defaultBackground = incomingDefaults.background;
+        }
+    }
+    const colorMap = new ColorMap();
+    // start with token colors from custom token themes
+    for (const color of customTokenColors) {
+        colorMap.getId(color);
+    }
+    const foregroundColorId = colorMap.getId(defaultForeground);
+    const backgroundColorId = colorMap.getId(defaultBackground);
+    const defaults = new ThemeTrieElementRule(defaultFontStyle, foregroundColorId, backgroundColorId);
+    const root = new ThemeTrieElement(defaults);
+    for (let i = 0, len = parsedThemeRules.length; i < len; i++) {
+        const rule = parsedThemeRules[i];
+        root.insert(rule.token, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background));
+    }
+    return new TokenTheme(colorMap, root);
+}
+const colorRegExp = /^#?([0-9A-Fa-f]{6})([0-9A-Fa-f]{2})?$/;
+class ColorMap {
+    constructor() {
+        this._lastColorId = 0;
+        this._id2color = [];
+        this._color2id = new Map();
+    }
+    getId(color) {
+        if (color === null) {
+            return 0;
+        }
+        const match = color.match(colorRegExp);
+        if (!match) {
+            throw new Error('Illegal value for token color: ' + color);
+        }
+        color = match[1].toUpperCase();
+        let value = this._color2id.get(color);
+        if (value) {
+            return value;
+        }
+        value = ++this._lastColorId;
+        this._color2id.set(color, value);
+        this._id2color[value] = Color.fromHex('#' + color);
+        return value;
+    }
+    getColorMap() {
+        return this._id2color.slice(0);
+    }
+}
+class TokenTheme {
+    static createFromRawTokenTheme(source, customTokenColors) {
+        return this.createFromParsedTokenTheme(parseTokenTheme(source), customTokenColors);
+    }
+    static createFromParsedTokenTheme(source, customTokenColors) {
+        return resolveParsedTokenThemeRules(source, customTokenColors);
+    }
+    constructor(colorMap, root) {
+        this._colorMap = colorMap;
+        this._root = root;
+        this._cache = new Map();
+    }
+    getColorMap() {
+        return this._colorMap.getColorMap();
+    }
+    _match(token) {
+        return this._root.match(token);
+    }
+    match(languageId, token) {
+        // The cache contains the metadata without the language bits set.
+        let result = this._cache.get(token);
+        if (typeof result === 'undefined') {
+            const rule = this._match(token);
+            const standardToken = toStandardTokenType(token);
+            result = (rule.metadata
+                | (standardToken << 8 /* MetadataConsts.TOKEN_TYPE_OFFSET */)) >>> 0;
+            this._cache.set(token, result);
+        }
+        return (result
+            | (languageId << 0 /* MetadataConsts.LANGUAGEID_OFFSET */)) >>> 0;
+    }
+}
+const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|regexp)\b/;
+function toStandardTokenType(tokenType) {
+    const m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
+    if (!m) {
+        return 0 /* StandardTokenType.Other */;
+    }
+    switch (m[1]) {
+        case 'comment':
+            return 1 /* StandardTokenType.Comment */;
+        case 'string':
+            return 2 /* StandardTokenType.String */;
+        case 'regex':
+            return 3 /* StandardTokenType.RegEx */;
+        case 'regexp':
+            return 3 /* StandardTokenType.RegEx */;
+    }
+    throw new Error('Unexpected match for standard token type!');
+}
+function strcmp(a, b) {
+    if (a < b) {
+        return -1;
+    }
+    if (a > b) {
+        return 1;
+    }
+    return 0;
+}
+class ThemeTrieElementRule {
+    constructor(fontStyle, foreground, background) {
+        this._themeTrieElementRuleBrand = undefined;
+        this._fontStyle = fontStyle;
+        this._foreground = foreground;
+        this._background = background;
+        this.metadata = ((this._fontStyle << 11 /* MetadataConsts.FONT_STYLE_OFFSET */)
+            | (this._foreground << 15 /* MetadataConsts.FOREGROUND_OFFSET */)
+            | (this._background << 24 /* MetadataConsts.BACKGROUND_OFFSET */)) >>> 0;
+    }
+    clone() {
+        return new ThemeTrieElementRule(this._fontStyle, this._foreground, this._background);
+    }
+    acceptOverwrite(fontStyle, foreground, background) {
+        if (fontStyle !== -1 /* FontStyle.NotSet */) {
+            this._fontStyle = fontStyle;
+        }
+        if (foreground !== 0 /* ColorId.None */) {
+            this._foreground = foreground;
+        }
+        if (background !== 0 /* ColorId.None */) {
+            this._background = background;
+        }
+        this.metadata = ((this._fontStyle << 11 /* MetadataConsts.FONT_STYLE_OFFSET */)
+            | (this._foreground << 15 /* MetadataConsts.FOREGROUND_OFFSET */)
+            | (this._background << 24 /* MetadataConsts.BACKGROUND_OFFSET */)) >>> 0;
+    }
+}
+class ThemeTrieElement {
+    constructor(mainRule) {
+        this._themeTrieElementBrand = undefined;
+        this._mainRule = mainRule;
+        this._children = new Map();
+    }
+    match(token) {
+        if (token === '') {
+            return this._mainRule;
+        }
+        const dotIndex = token.indexOf('.');
+        let head;
+        let tail;
+        if (dotIndex === -1) {
+            head = token;
+            tail = '';
+        }
+        else {
+            head = token.substring(0, dotIndex);
+            tail = token.substring(dotIndex + 1);
+        }
+        const child = this._children.get(head);
+        if (typeof child !== 'undefined') {
+            return child.match(tail);
+        }
+        return this._mainRule;
+    }
+    insert(token, fontStyle, foreground, background) {
+        if (token === '') {
+            // Merge into the main rule
+            this._mainRule.acceptOverwrite(fontStyle, foreground, background);
+            return;
+        }
+        const dotIndex = token.indexOf('.');
+        let head;
+        let tail;
+        if (dotIndex === -1) {
+            head = token;
+            tail = '';
+        }
+        else {
+            head = token.substring(0, dotIndex);
+            tail = token.substring(dotIndex + 1);
+        }
+        let child = this._children.get(head);
+        if (typeof child === 'undefined') {
+            child = new ThemeTrieElement(this._mainRule.clone());
+            this._children.set(head, child);
+        }
+        child.insert(tail, fontStyle, foreground, background);
+    }
+}
+function generateTokensCSSForColorMap(colorMap) {
+    const rules = [];
+    for (let i = 1, len = colorMap.length; i < len; i++) {
+        const color = colorMap[i];
+        rules[i] = `.mtk${i} { color: ${color}; }`;
+    }
+    rules.push('.mtki { font-style: italic; }');
+    rules.push('.mtkb { font-weight: bold; }');
+    rules.push('.mtku { text-decoration: underline; text-underline-position: under; }');
+    rules.push('.mtks { text-decoration: line-through; }');
+    rules.push('.mtks.mtku { text-decoration: underline line-through; text-underline-position: under; }');
+    return rules.join('\n');
+}
+
+export { ColorMap, ParsedTokenThemeRule, ThemeTrieElement, ThemeTrieElementRule, TokenTheme, generateTokensCSSForColorMap, parseTokenTheme, strcmp, toStandardTokenType };

package/dist/workers/editor/common/tokenizationRegistry.js

@@ -0,0 +1,123 @@
+import { Emitter } from '../../base/common/event.js';
+import { toDisposable, Disposable } from '../../base/common/lifecycle.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+class TokenizationRegistry {
+    constructor() {
+        this._tokenizationSupports = new Map();
+        this._factories = new Map();
+        this._onDidChange = new Emitter();
+        this.onDidChange = this._onDidChange.event;
+        this._colorMap = null;
+    }
+    handleChange(languageIds) {
+        this._onDidChange.fire({
+            changedLanguages: languageIds,
+            changedColorMap: false
+        });
+    }
+    register(languageId, support) {
+        this._tokenizationSupports.set(languageId, support);
+        this.handleChange([languageId]);
+        return toDisposable(() => {
+            if (this._tokenizationSupports.get(languageId) !== support) {
+                return;
+            }
+            this._tokenizationSupports.delete(languageId);
+            this.handleChange([languageId]);
+        });
+    }
+    get(languageId) {
+        return this._tokenizationSupports.get(languageId) || null;
+    }
+    registerFactory(languageId, factory) {
+        this._factories.get(languageId)?.dispose();
+        const myData = new TokenizationSupportFactoryData(this, languageId, factory);
+        this._factories.set(languageId, myData);
+        return toDisposable(() => {
+            const v = this._factories.get(languageId);
+            if (!v || v !== myData) {
+                return;
+            }
+            this._factories.delete(languageId);
+            v.dispose();
+        });
+    }
+    async getOrCreate(languageId) {
+        // check first if the support is already set
+        const tokenizationSupport = this.get(languageId);
+        if (tokenizationSupport) {
+            return tokenizationSupport;
+        }
+        const factory = this._factories.get(languageId);
+        if (!factory || factory.isResolved) {
+            // no factory or factory.resolve already finished
+            return null;
+        }
+        await factory.resolve();
+        return this.get(languageId);
+    }
+    isResolved(languageId) {
+        const tokenizationSupport = this.get(languageId);
+        if (tokenizationSupport) {
+            return true;
+        }
+        const factory = this._factories.get(languageId);
+        if (!factory || factory.isResolved) {
+            return true;
+        }
+        return false;
+    }
+    setColorMap(colorMap) {
+        this._colorMap = colorMap;
+        this._onDidChange.fire({
+            changedLanguages: Array.from(this._tokenizationSupports.keys()),
+            changedColorMap: true
+        });
+    }
+    getColorMap() {
+        return this._colorMap;
+    }
+    getDefaultBackground() {
+        if (this._colorMap && this._colorMap.length > 2 /* ColorId.DefaultBackground */) {
+            return this._colorMap[2 /* ColorId.DefaultBackground */];
+        }
+        return null;
+    }
+}
+class TokenizationSupportFactoryData extends Disposable {
+    get isResolved() {
+        return this._isResolved;
+    }
+    constructor(_registry, _languageId, _factory) {
+        super();
+        this._registry = _registry;
+        this._languageId = _languageId;
+        this._factory = _factory;
+        this._isDisposed = false;
+        this._resolvePromise = null;
+        this._isResolved = false;
+    }
+    dispose() {
+        this._isDisposed = true;
+        super.dispose();
+    }
+    async resolve() {
+        if (!this._resolvePromise) {
+            this._resolvePromise = this._create();
+        }
+        return this._resolvePromise;
+    }
+    async _create() {
+        const value = await this._factory.tokenizationSupport;
+        this._isResolved = true;
+        if (value && !this._isDisposed) {
+            this._register(this._registry.register(this._languageId, value));
+        }
+    }
+}
+
+export { TokenizationRegistry };

package/dist/workers/editor/common/tokenizationTextModelPart.js

@@ -0,0 +1,275 @@
+import { BugIndicatingError } from '../../../../base/common/errors.js';
+import { Emitter } from '../../../../base/common/event.js';
+import { countEOL } from '../../core/misc/eolCounter.js';
+import { Position } from '../../core/position.js';
+import { getWordAtText } from '../../core/wordHelper.js';
+import { ILanguageService } from '../../languages/language.js';
+import { ILanguageConfigurationService } from '../../languages/languageConfigurationRegistry.js';
+import { TextModelPart } from '../textModelPart.js';
+import { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';
+import { SparseTokensStore } from '../../tokens/sparseTokensStore.js';
+import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
+import { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';
+import { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';
+import '../../../../base/common/observableInternal/index.js';
+import { observableValue } from '../../../../base/common/observableInternal/observables/observableValue.js';
+import { derived } from '../../../../base/common/observableInternal/observables/derived.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+var __decorate = (undefined && undefined.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (undefined && undefined.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var TokenizationTextModelPart_1;
+let TokenizationTextModelPart = TokenizationTextModelPart_1 = class TokenizationTextModelPart extends TextModelPart {
+    constructor(_textModel, _bracketPairsTextModelPart, _languageId, _attachedViews, _languageService, _languageConfigurationService, _instantiationService, _treeSitterLibraryService) {
+        super();
+        this._textModel = _textModel;
+        this._bracketPairsTextModelPart = _bracketPairsTextModelPart;
+        this._languageId = _languageId;
+        this._attachedViews = _attachedViews;
+        this._languageService = _languageService;
+        this._languageConfigurationService = _languageConfigurationService;
+        this._instantiationService = _instantiationService;
+        this._treeSitterLibraryService = _treeSitterLibraryService;
+        this._languageIdObs = observableValue(this, this._languageId);
+        this._useTreeSitter = derived(this, reader => {
+            const languageId = this._languageIdObs.read(reader);
+            return this._treeSitterLibraryService.supportsLanguage(languageId, reader);
+        });
+        this.tokens = derived(this, reader => {
+            let tokens;
+            if (this._useTreeSitter.read(reader)) {
+                tokens = reader.store.add(this._instantiationService.createInstance(TreeSitterSyntaxTokenBackend, this._languageIdObs, this._languageService.languageIdCodec, this._textModel, this._attachedViews.visibleLineRanges));
+            }
+            else {
+                tokens = reader.store.add(new TokenizerSyntaxTokenBackend(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews));
+            }
+            reader.store.add(tokens.onDidChangeTokens(e => {
+                this._emitModelTokensChangedEvent(e);
+            }));
+            reader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {
+                this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
+            }));
+            return tokens;
+        });
+        let hadTokens = false;
+        this.tokens.recomputeInitiallyAndOnChange(this._store, value => {
+            if (hadTokens) {
+                // We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor.
+                // TODO@hediet: Look into why this is needed.
+                value.todo_resetTokenization();
+            }
+            hadTokens = true;
+        });
+        this._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);
+        this._onDidChangeLanguage = this._register(new Emitter());
+        this.onDidChangeLanguage = this._onDidChangeLanguage.event;
+        this._onDidChangeLanguageConfiguration = this._register(new Emitter());
+        this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
+        this._onDidChangeTokens = this._register(new Emitter());
+        this.onDidChangeTokens = this._onDidChangeTokens.event;
+    }
+    handleLanguageConfigurationServiceChange(e) {
+        if (e.affects(this._languageId)) {
+            this._onDidChangeLanguageConfiguration.fire({});
+        }
+    }
+    handleDidChangeContent(e) {
+        if (e.isFlush) {
+            this._semanticTokens.flush();
+        }
+        else if (!e.isEolChange) { // We don't have to do anything on an EOL change
+            for (const c of e.changes) {
+                const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
+                this._semanticTokens.acceptEdit(c.range, eolCount, firstLineLength, lastLineLength, c.text.length > 0 ? c.text.charCodeAt(0) : 0 /* CharCode.Null */);
+            }
+        }
+        this.tokens.get().handleDidChangeContent(e);
+    }
+    handleDidChangeAttached() {
+        this.tokens.get().handleDidChangeAttached();
+    }
+    /**
+     * Includes grammar and semantic tokens.
+     */
+    getLineTokens(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        const syntacticTokens = this.tokens.get().getLineTokens(lineNumber);
+        return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
+    }
+    _emitModelTokensChangedEvent(e) {
+        if (!this._textModel._isDisposing()) {
+            this._bracketPairsTextModelPart.handleDidChangeTokens(e);
+            this._onDidChangeTokens.fire(e);
+        }
+    }
+    // #region Grammar Tokens
+    validateLineNumber(lineNumber) {
+        if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
+            throw new BugIndicatingError('Illegal value for lineNumber');
+        }
+    }
+    get hasTokens() {
+        return this.tokens.get().hasTokens;
+    }
+    resetTokenization() {
+        this.tokens.get().todo_resetTokenization();
+    }
+    get backgroundTokenizationState() {
+        return this.tokens.get().backgroundTokenizationState;
+    }
+    forceTokenization(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        this.tokens.get().forceTokenization(lineNumber);
+    }
+    hasAccurateTokensForLine(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        return this.tokens.get().hasAccurateTokensForLine(lineNumber);
+    }
+    isCheapToTokenize(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        return this.tokens.get().isCheapToTokenize(lineNumber);
+    }
+    tokenizeIfCheap(lineNumber) {
+        this.validateLineNumber(lineNumber);
+        this.tokens.get().tokenizeIfCheap(lineNumber);
+    }
+    getTokenTypeIfInsertingCharacter(lineNumber, column, character) {
+        return this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);
+    }
+    tokenizeLinesAt(lineNumber, lines) {
+        return this.tokens.get().tokenizeLinesAt(lineNumber, lines);
+    }
+    // #endregion
+    // #region Semantic Tokens
+    setSemanticTokens(tokens, isComplete) {
+        this._semanticTokens.set(tokens, isComplete, this._textModel);
+        this._emitModelTokensChangedEvent({
+            semanticTokensApplied: tokens !== null,
+            ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],
+        });
+    }
+    hasCompleteSemanticTokens() {
+        return this._semanticTokens.isComplete();
+    }
+    hasSomeSemanticTokens() {
+        return !this._semanticTokens.isEmpty();
+    }
+    setPartialSemanticTokens(range, tokens) {
+        if (this.hasCompleteSemanticTokens()) {
+            return;
+        }
+        const changedRange = this._textModel.validateRange(this._semanticTokens.setPartial(range, tokens));
+        this._emitModelTokensChangedEvent({
+            semanticTokensApplied: true,
+            ranges: [
+                {
+                    fromLineNumber: changedRange.startLineNumber,
+                    toLineNumber: changedRange.endLineNumber,
+                },
+            ],
+        });
+    }
+    // #endregion
+    // #region Utility Methods
+    getWordAtPosition(_position) {
+        this.assertNotDisposed();
+        const position = this._textModel.validatePosition(_position);
+        const lineContent = this._textModel.getLineContent(position.lineNumber);
+        const lineTokens = this.getLineTokens(position.lineNumber);
+        const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
+        // (1). First try checking right biased word
+        const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex);
+        const rightBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(), lineContent.substring(rbStartOffset, rbEndOffset), rbStartOffset);
+        // Make sure the result touches the original passed in position
+        if (rightBiasedWord &&
+            rightBiasedWord.startColumn <= _position.column &&
+            _position.column <= rightBiasedWord.endColumn) {
+            return rightBiasedWord;
+        }
+        // (2). Else, if we were at a language boundary, check the left biased word
+        if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
+            // edge case, where `position` sits between two tokens belonging to two different languages
+            const [lbStartOffset, lbEndOffset] = TokenizationTextModelPart_1._findLanguageBoundaries(lineTokens, tokenIndex - 1);
+            const leftBiasedWord = getWordAtText(position.column, this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(), lineContent.substring(lbStartOffset, lbEndOffset), lbStartOffset);
+            // Make sure the result touches the original passed in position
+            if (leftBiasedWord &&
+                leftBiasedWord.startColumn <= _position.column &&
+                _position.column <= leftBiasedWord.endColumn) {
+                return leftBiasedWord;
+            }
+        }
+        return null;
+    }
+    getLanguageConfiguration(languageId) {
+        return this._languageConfigurationService.getLanguageConfiguration(languageId);
+    }
+    static _findLanguageBoundaries(lineTokens, tokenIndex) {
+        const languageId = lineTokens.getLanguageId(tokenIndex);
+        // go left until a different language is hit
+        let startOffset = 0;
+        for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {
+            startOffset = lineTokens.getStartOffset(i);
+        }
+        // go right until a different language is hit
+        let endOffset = lineTokens.getLineContent().length;
+        for (let i = tokenIndex, tokenCount = lineTokens.getCount(); i < tokenCount && lineTokens.getLanguageId(i) === languageId; i++) {
+            endOffset = lineTokens.getEndOffset(i);
+        }
+        return [startOffset, endOffset];
+    }
+    getWordUntilPosition(position) {
+        const wordAtPosition = this.getWordAtPosition(position);
+        if (!wordAtPosition) {
+            return { word: '', startColumn: position.column, endColumn: position.column, };
+        }
+        return {
+            word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
+            startColumn: wordAtPosition.startColumn,
+            endColumn: position.column,
+        };
+    }
+    // #endregion
+    // #region Language Id handling
+    getLanguageId() {
+        return this._languageId;
+    }
+    getLanguageIdAtPosition(lineNumber, column) {
+        const position = this._textModel.validatePosition(new Position(lineNumber, column));
+        const lineTokens = this.getLineTokens(position.lineNumber);
+        return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
+    }
+    setLanguageId(languageId, source = 'api') {
+        if (this._languageId === languageId) {
+            // There's nothing to do
+            return;
+        }
+        const e = {
+            oldLanguage: this._languageId,
+            newLanguage: languageId,
+            source
+        };
+        this._languageId = languageId;
+        this._languageIdObs.set(languageId, undefined);
+        this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
+        this._onDidChangeLanguage.fire(e);
+        this._onDidChangeLanguageConfiguration.fire({});
+    }
+};
+TokenizationTextModelPart = TokenizationTextModelPart_1 = __decorate([
+    __param(4, ILanguageService),
+    __param(5, ILanguageConfigurationService),
+    __param(6, IInstantiationService),
+    __param(7, ITreeSitterLibraryService)
+], TokenizationTextModelPart);
+
+export { TokenizationTextModelPart };