monaco-editor11 1.1.0 → 1.1.2

package/dist/workers/textModelTokens.js

@@ -0,0 +1,436 @@
+import { runWhenGlobalIdle } from '../../../base/common/async.js';
+import { BugIndicatingError, onUnexpectedError } from '../../../base/common/errors.js';
+import { setTimeout0 } from '../../../base/common/platform.js';
+import { StopWatch } from '../../../base/common/stopwatch.js';
+import { countEOL } from '../core/misc/eolCounter.js';
+import { LineRange } from '../core/ranges/lineRange.js';
+import { OffsetRange } from '../core/ranges/offsetRange.js';
+import { nullTokenizeEncoded } from '../languages/nullTokenize.js';
+import { FixedArray } from './fixedArray.js';
+import { ContiguousMultilineTokensBuilder } from '../tokens/contiguousMultilineTokensBuilder.js';
+import { LineTokens } from '../tokens/lineTokens.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+class TokenizerWithStateStore {
+    constructor(lineCount, tokenizationSupport) {
+        this.tokenizationSupport = tokenizationSupport;
+        this.initialState = this.tokenizationSupport.getInitialState();
+        this.store = new TrackingTokenizationStateStore(lineCount);
+    }
+    getStartState(lineNumber) {
+        return this.store.getStartState(lineNumber, this.initialState);
+    }
+    getFirstInvalidLine() {
+        return this.store.getFirstInvalidLine(this.initialState);
+    }
+}
+class TokenizerWithStateStoreAndTextModel extends TokenizerWithStateStore {
+    constructor(lineCount, tokenizationSupport, _textModel, _languageIdCodec) {
+        super(lineCount, tokenizationSupport);
+        this._textModel = _textModel;
+        this._languageIdCodec = _languageIdCodec;
+    }
+    updateTokensUntilLine(builder, lineNumber) {
+        const languageId = this._textModel.getLanguageId();
+        while (true) {
+            const lineToTokenize = this.getFirstInvalidLine();
+            if (!lineToTokenize || lineToTokenize.lineNumber > lineNumber) {
+                break;
+            }
+            const text = this._textModel.getLineContent(lineToTokenize.lineNumber);
+            const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineToTokenize.startState);
+            builder.add(lineToTokenize.lineNumber, r.tokens);
+            this.store.setEndState(lineToTokenize.lineNumber, r.endState);
+        }
+    }
+    /** assumes state is up to date */
+    getTokenTypeIfInsertingCharacter(position, character) {
+        // TODO@hediet: use tokenizeLineWithEdit
+        const lineStartState = this.getStartState(position.lineNumber);
+        if (!lineStartState) {
+            return 0 /* StandardTokenType.Other */;
+        }
+        const languageId = this._textModel.getLanguageId();
+        const lineContent = this._textModel.getLineContent(position.lineNumber);
+        // Create the text as if `character` was inserted
+        const text = (lineContent.substring(0, position.column - 1)
+            + character
+            + lineContent.substring(position.column - 1));
+        const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState);
+        const lineTokens = new LineTokens(r.tokens, text, this._languageIdCodec);
+        if (lineTokens.getCount() === 0) {
+            return 0 /* StandardTokenType.Other */;
+        }
+        const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
+        return lineTokens.getStandardTokenType(tokenIndex);
+    }
+    /** assumes state is up to date */
+    tokenizeLinesAt(lineNumber, lines) {
+        const lineStartState = this.getStartState(lineNumber);
+        if (!lineStartState) {
+            return null;
+        }
+        const languageId = this._textModel.getLanguageId();
+        const result = [];
+        let state = lineStartState;
+        for (const line of lines) {
+            const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, line, true, state);
+            result.push(new LineTokens(r.tokens, line, this._languageIdCodec));
+            state = r.endState;
+        }
+        return result;
+    }
+    hasAccurateTokensForLine(lineNumber) {
+        const firstInvalidLineNumber = this.store.getFirstInvalidEndStateLineNumberOrMax();
+        return (lineNumber < firstInvalidLineNumber);
+    }
+    isCheapToTokenize(lineNumber) {
+        const firstInvalidLineNumber = this.store.getFirstInvalidEndStateLineNumberOrMax();
+        if (lineNumber < firstInvalidLineNumber) {
+            return true;
+        }
+        if (lineNumber === firstInvalidLineNumber
+            && this._textModel.getLineLength(lineNumber) < 2048 /* Constants.CHEAP_TOKENIZATION_LENGTH_LIMIT */) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * The result is not cached.
+     */
+    tokenizeHeuristically(builder, startLineNumber, endLineNumber) {
+        if (endLineNumber <= this.store.getFirstInvalidEndStateLineNumberOrMax()) {
+            // nothing to do
+            return { heuristicTokens: false };
+        }
+        if (startLineNumber <= this.store.getFirstInvalidEndStateLineNumberOrMax()) {
+            // tokenization has reached the viewport start...
+            this.updateTokensUntilLine(builder, endLineNumber);
+            return { heuristicTokens: false };
+        }
+        let state = this.guessStartState(startLineNumber);
+        const languageId = this._textModel.getLanguageId();
+        for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
+            const text = this._textModel.getLineContent(lineNumber);
+            const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, state);
+            builder.add(lineNumber, r.tokens);
+            state = r.endState;
+        }
+        return { heuristicTokens: true };
+    }
+    guessStartState(lineNumber) {
+        let { likelyRelevantLines, initialState } = findLikelyRelevantLines(this._textModel, lineNumber, this);
+        if (!initialState) {
+            initialState = this.tokenizationSupport.getInitialState();
+        }
+        const languageId = this._textModel.getLanguageId();
+        let state = initialState;
+        for (const line of likelyRelevantLines) {
+            const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, line, false, state);
+            state = r.endState;
+        }
+        return state;
+    }
+}
+function findLikelyRelevantLines(model, lineNumber, store) {
+    let nonWhitespaceColumn = model.getLineFirstNonWhitespaceColumn(lineNumber);
+    const likelyRelevantLines = [];
+    let initialState = null;
+    for (let i = lineNumber - 1; nonWhitespaceColumn > 1 && i >= 1; i--) {
+        const newNonWhitespaceIndex = model.getLineFirstNonWhitespaceColumn(i);
+        // Ignore lines full of whitespace
+        if (newNonWhitespaceIndex === 0) {
+            continue;
+        }
+        if (newNonWhitespaceIndex < nonWhitespaceColumn) {
+            likelyRelevantLines.push(model.getLineContent(i));
+            nonWhitespaceColumn = newNonWhitespaceIndex;
+            initialState = store?.getStartState(i);
+            if (initialState) {
+                break;
+            }
+        }
+    }
+    likelyRelevantLines.reverse();
+    return { likelyRelevantLines, initialState: initialState ?? undefined };
+}
+/**
+ * **Invariant:**
+ * If the text model is retokenized from line 1 to {@link getFirstInvalidEndStateLineNumber}() - 1,
+ * then the recomputed end state for line l will be equal to {@link getEndState}(l).
+ */
+class TrackingTokenizationStateStore {
+    constructor(lineCount) {
+        this.lineCount = lineCount;
+        this._tokenizationStateStore = new TokenizationStateStore();
+        this._invalidEndStatesLineNumbers = new RangePriorityQueueImpl();
+        this._invalidEndStatesLineNumbers.addRange(new OffsetRange(1, lineCount + 1));
+    }
+    getEndState(lineNumber) {
+        return this._tokenizationStateStore.getEndState(lineNumber);
+    }
+    /**
+     * @returns if the end state has changed.
+     */
+    setEndState(lineNumber, state) {
+        if (!state) {
+            throw new BugIndicatingError('Cannot set null/undefined state');
+        }
+        this._invalidEndStatesLineNumbers.delete(lineNumber);
+        const r = this._tokenizationStateStore.setEndState(lineNumber, state);
+        if (r && lineNumber < this.lineCount) {
+            // because the state changed, we cannot trust the next state anymore and have to invalidate it.
+            this._invalidEndStatesLineNumbers.addRange(new OffsetRange(lineNumber + 1, lineNumber + 2));
+        }
+        return r;
+    }
+    acceptChange(range, newLineCount) {
+        this.lineCount += newLineCount - range.length;
+        this._tokenizationStateStore.acceptChange(range, newLineCount);
+        this._invalidEndStatesLineNumbers.addRangeAndResize(new OffsetRange(range.startLineNumber, range.endLineNumberExclusive), newLineCount);
+    }
+    acceptChanges(changes) {
+        for (const c of changes) {
+            const [eolCount] = countEOL(c.text);
+            this.acceptChange(new LineRange(c.range.startLineNumber, c.range.endLineNumber + 1), eolCount + 1);
+        }
+    }
+    invalidateEndStateRange(range) {
+        this._invalidEndStatesLineNumbers.addRange(new OffsetRange(range.startLineNumber, range.endLineNumberExclusive));
+    }
+    getFirstInvalidEndStateLineNumber() { return this._invalidEndStatesLineNumbers.min; }
+    getFirstInvalidEndStateLineNumberOrMax() {
+        return this.getFirstInvalidEndStateLineNumber() || Number.MAX_SAFE_INTEGER;
+    }
+    allStatesValid() { return this._invalidEndStatesLineNumbers.min === null; }
+    getStartState(lineNumber, initialState) {
+        if (lineNumber === 1) {
+            return initialState;
+        }
+        return this.getEndState(lineNumber - 1);
+    }
+    getFirstInvalidLine(initialState) {
+        const lineNumber = this.getFirstInvalidEndStateLineNumber();
+        if (lineNumber === null) {
+            return null;
+        }
+        const startState = this.getStartState(lineNumber, initialState);
+        if (!startState) {
+            throw new BugIndicatingError('Start state must be defined');
+        }
+        return { lineNumber, startState };
+    }
+}
+class TokenizationStateStore {
+    constructor() {
+        this._lineEndStates = new FixedArray(null);
+    }
+    getEndState(lineNumber) {
+        return this._lineEndStates.get(lineNumber);
+    }
+    setEndState(lineNumber, state) {
+        const oldState = this._lineEndStates.get(lineNumber);
+        if (oldState && oldState.equals(state)) {
+            return false;
+        }
+        this._lineEndStates.set(lineNumber, state);
+        return true;
+    }
+    acceptChange(range, newLineCount) {
+        let length = range.length;
+        if (newLineCount > 0 && length > 0) {
+            // Keep the last state, even though it is unrelated.
+            // But if the new state happens to agree with this last state, then we know we can stop tokenizing.
+            length--;
+            newLineCount--;
+        }
+        this._lineEndStates.replace(range.startLineNumber, length, newLineCount);
+    }
+}
+class RangePriorityQueueImpl {
+    constructor() {
+        this._ranges = [];
+    }
+    get min() {
+        if (this._ranges.length === 0) {
+            return null;
+        }
+        return this._ranges[0].start;
+    }
+    delete(value) {
+        const idx = this._ranges.findIndex(r => r.contains(value));
+        if (idx !== -1) {
+            const range = this._ranges[idx];
+            if (range.start === value) {
+                if (range.endExclusive === value + 1) {
+                    this._ranges.splice(idx, 1);
+                }
+                else {
+                    this._ranges[idx] = new OffsetRange(value + 1, range.endExclusive);
+                }
+            }
+            else {
+                if (range.endExclusive === value + 1) {
+                    this._ranges[idx] = new OffsetRange(range.start, value);
+                }
+                else {
+                    this._ranges.splice(idx, 1, new OffsetRange(range.start, value), new OffsetRange(value + 1, range.endExclusive));
+                }
+            }
+        }
+    }
+    addRange(range) {
+        OffsetRange.addRange(range, this._ranges);
+    }
+    addRangeAndResize(range, newLength) {
+        let idxFirstMightBeIntersecting = 0;
+        while (!(idxFirstMightBeIntersecting >= this._ranges.length || range.start <= this._ranges[idxFirstMightBeIntersecting].endExclusive)) {
+            idxFirstMightBeIntersecting++;
+        }
+        let idxFirstIsAfter = idxFirstMightBeIntersecting;
+        while (!(idxFirstIsAfter >= this._ranges.length || range.endExclusive < this._ranges[idxFirstIsAfter].start)) {
+            idxFirstIsAfter++;
+        }
+        const delta = newLength - range.length;
+        for (let i = idxFirstIsAfter; i < this._ranges.length; i++) {
+            this._ranges[i] = this._ranges[i].delta(delta);
+        }
+        if (idxFirstMightBeIntersecting === idxFirstIsAfter) {
+            const newRange = new OffsetRange(range.start, range.start + newLength);
+            if (!newRange.isEmpty) {
+                this._ranges.splice(idxFirstMightBeIntersecting, 0, newRange);
+            }
+        }
+        else {
+            const start = Math.min(range.start, this._ranges[idxFirstMightBeIntersecting].start);
+            const endEx = Math.max(range.endExclusive, this._ranges[idxFirstIsAfter - 1].endExclusive);
+            const newRange = new OffsetRange(start, endEx + delta);
+            if (!newRange.isEmpty) {
+                this._ranges.splice(idxFirstMightBeIntersecting, idxFirstIsAfter - idxFirstMightBeIntersecting, newRange);
+            }
+            else {
+                this._ranges.splice(idxFirstMightBeIntersecting, idxFirstIsAfter - idxFirstMightBeIntersecting);
+            }
+        }
+    }
+    toString() {
+        return this._ranges.map(r => r.toString()).join(' + ');
+    }
+}
+function safeTokenize(languageIdCodec, languageId, tokenizationSupport, text, hasEOL, state) {
+    let r = null;
+    if (tokenizationSupport) {
+        try {
+            r = tokenizationSupport.tokenizeEncoded(text, hasEOL, state.clone());
+        }
+        catch (e) {
+            onUnexpectedError(e);
+        }
+    }
+    if (!r) {
+        r = nullTokenizeEncoded(languageIdCodec.encodeLanguageId(languageId), state);
+    }
+    LineTokens.convertToEndOffset(r.tokens, text.length);
+    return r;
+}
+class DefaultBackgroundTokenizer {
+    constructor(_tokenizerWithStateStore, _backgroundTokenStore) {
+        this._tokenizerWithStateStore = _tokenizerWithStateStore;
+        this._backgroundTokenStore = _backgroundTokenStore;
+        this._isDisposed = false;
+        this._isScheduled = false;
+    }
+    dispose() {
+        this._isDisposed = true;
+    }
+    handleChanges() {
+        this._beginBackgroundTokenization();
+    }
+    _beginBackgroundTokenization() {
+        if (this._isScheduled || !this._tokenizerWithStateStore._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
+            return;
+        }
+        this._isScheduled = true;
+        runWhenGlobalIdle((deadline) => {
+            this._isScheduled = false;
+            this._backgroundTokenizeWithDeadline(deadline);
+        });
+    }
+    /**
+     * Tokenize until the deadline occurs, but try to yield every 1-2ms.
+     */
+    _backgroundTokenizeWithDeadline(deadline) {
+        // Read the time remaining from the `deadline` immediately because it is unclear
+        // if the `deadline` object will be valid after execution leaves this function.
+        const endTime = Date.now() + deadline.timeRemaining();
+        const execute = () => {
+            if (this._isDisposed || !this._tokenizerWithStateStore._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
+                // disposed in the meantime or detached or finished
+                return;
+            }
+            this._backgroundTokenizeForAtLeast1ms();
+            if (Date.now() < endTime) {
+                // There is still time before reaching the deadline, so yield to the browser and then
+                // continue execution
+                setTimeout0(execute);
+            }
+            else {
+                // The deadline has been reached, so schedule a new idle callback if necessary
+                this._beginBackgroundTokenization();
+            }
+        };
+        execute();
+    }
+    /**
+     * Tokenize for at least 1ms.
+     */
+    _backgroundTokenizeForAtLeast1ms() {
+        const lineCount = this._tokenizerWithStateStore._textModel.getLineCount();
+        const builder = new ContiguousMultilineTokensBuilder();
+        const sw = StopWatch.create(false);
+        do {
+            if (sw.elapsed() > 1) {
+                // the comparison is intentionally > 1 and not >= 1 to ensure that
+                // a full millisecond has elapsed, given how microseconds are rounded
+                // to milliseconds
+                break;
+            }
+            const tokenizedLineNumber = this._tokenizeOneInvalidLine(builder);
+            if (tokenizedLineNumber >= lineCount) {
+                break;
+            }
+        } while (this._hasLinesToTokenize());
+        this._backgroundTokenStore.setTokens(builder.finalize());
+        this.checkFinished();
+    }
+    _hasLinesToTokenize() {
+        if (!this._tokenizerWithStateStore) {
+            return false;
+        }
+        return !this._tokenizerWithStateStore.store.allStatesValid();
+    }
+    _tokenizeOneInvalidLine(builder) {
+        const firstInvalidLine = this._tokenizerWithStateStore?.getFirstInvalidLine();
+        if (!firstInvalidLine) {
+            return this._tokenizerWithStateStore._textModel.getLineCount() + 1;
+        }
+        this._tokenizerWithStateStore.updateTokensUntilLine(builder, firstInvalidLine.lineNumber);
+        return firstInvalidLine.lineNumber;
+    }
+    checkFinished() {
+        if (this._isDisposed) {
+            return;
+        }
+        if (this._tokenizerWithStateStore.store.allStatesValid()) {
+            this._backgroundTokenStore.backgroundTokenizationFinished();
+        }
+    }
+    requestTokens(startLineNumber, endLineNumberExclusive) {
+        this._tokenizerWithStateStore.store.invalidateEndStateRange(new LineRange(startLineNumber, endLineNumberExclusive));
+    }
+}
+
+export { DefaultBackgroundTokenizer, RangePriorityQueueImpl, TokenizationStateStore, TokenizerWithStateStore, TokenizerWithStateStoreAndTextModel, TrackingTokenizationStateStore, findLikelyRelevantLines };

package/dist/workers/textResourceConfiguration.js

@@ -0,0 +1,6 @@
+import { createDecorator } from '../../../platform/instantiation/common/instantiation.js';
+
+const ITextResourceConfigurationService = createDecorator('textResourceConfigurationService');
+const ITextResourcePropertiesService = createDecorator('textResourcePropertiesService');
+
+export { ITextResourceConfigurationService, ITextResourcePropertiesService };

package/dist/workers/textToHtmlTokenizer.js

@@ -0,0 +1,139 @@
+import { splitLines, escape, isFullWidthCharacter } from '../../../base/common/strings.js';
+import { LineTokens } from '../tokens/lineTokens.js';
+import { TokenizationRegistry } from '../languages.js';
+import { nullTokenizeEncoded, NullState } from './nullTokenize.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+const fallback = {
+    getInitialState: () => NullState,
+    tokenizeEncoded: (buffer, hasEOL, state) => nullTokenizeEncoded(0 /* LanguageId.Null */, state)
+};
+async function tokenizeToString(languageService, text, languageId) {
+    if (!languageId) {
+        return _tokenizeToString(text, languageService.languageIdCodec, fallback);
+    }
+    const tokenizationSupport = await TokenizationRegistry.getOrCreate(languageId);
+    return _tokenizeToString(text, languageService.languageIdCodec, tokenizationSupport || fallback);
+}
+function tokenizeLineToHTML(text, viewLineTokens, colorMap, startOffset, endOffset, tabSize, useNbsp) {
+    let result = `<div>`;
+    let charIndex = 0;
+    let width = 0;
+    let prevIsSpace = true;
+    for (let tokenIndex = 0, tokenCount = viewLineTokens.getCount(); tokenIndex < tokenCount; tokenIndex++) {
+        const tokenEndIndex = viewLineTokens.getEndOffset(tokenIndex);
+        let partContent = '';
+        for (; charIndex < tokenEndIndex && charIndex < endOffset; charIndex++) {
+            const charCode = text.charCodeAt(charIndex);
+            const isTab = charCode === 9 /* CharCode.Tab */;
+            width += isFullWidthCharacter(charCode) ? 2 : (isTab ? 0 : 1);
+            if (charIndex < startOffset) {
+                if (isTab) {
+                    const remainder = width % tabSize;
+                    width += remainder === 0 ? tabSize : tabSize - remainder;
+                }
+                continue;
+            }
+            switch (charCode) {
+                case 9 /* CharCode.Tab */: {
+                    const remainder = width % tabSize;
+                    const insertSpacesCount = remainder === 0 ? tabSize : tabSize - remainder;
+                    width += insertSpacesCount;
+                    let spacesRemaining = insertSpacesCount;
+                    while (spacesRemaining > 0) {
+                        if (useNbsp && prevIsSpace) {
+                            partContent += '&#160;';
+                            prevIsSpace = false;
+                        }
+                        else {
+                            partContent += ' ';
+                            prevIsSpace = true;
+                        }
+                        spacesRemaining--;
+                    }
+                    break;
+                }
+                case 60 /* CharCode.LessThan */:
+                    partContent += '&lt;';
+                    prevIsSpace = false;
+                    break;
+                case 62 /* CharCode.GreaterThan */:
+                    partContent += '&gt;';
+                    prevIsSpace = false;
+                    break;
+                case 38 /* CharCode.Ampersand */:
+                    partContent += '&amp;';
+                    prevIsSpace = false;
+                    break;
+                case 0 /* CharCode.Null */:
+                    partContent += '&#00;';
+                    prevIsSpace = false;
+                    break;
+                case 65279 /* CharCode.UTF8_BOM */:
+                case 8232 /* CharCode.LINE_SEPARATOR */:
+                case 8233 /* CharCode.PARAGRAPH_SEPARATOR */:
+                case 133 /* CharCode.NEXT_LINE */:
+                    partContent += '\ufffd';
+                    prevIsSpace = false;
+                    break;
+                case 13 /* CharCode.CarriageReturn */:
+                    // zero width space, because carriage return would introduce a line break
+                    partContent += '&#8203';
+                    prevIsSpace = false;
+                    break;
+                case 32 /* CharCode.Space */:
+                    if (useNbsp && prevIsSpace) {
+                        partContent += '&#160;';
+                        prevIsSpace = false;
+                    }
+                    else {
+                        partContent += ' ';
+                        prevIsSpace = true;
+                    }
+                    break;
+                default:
+                    partContent += String.fromCharCode(charCode);
+                    prevIsSpace = false;
+            }
+        }
+        if (tokenEndIndex <= startOffset) {
+            continue;
+        }
+        result += `<span style="${viewLineTokens.getInlineStyle(tokenIndex, colorMap)}">${partContent}</span>`;
+        if (tokenEndIndex > endOffset || charIndex >= endOffset || startOffset >= endOffset) {
+            break;
+        }
+    }
+    result += `</div>`;
+    return result;
+}
+function _tokenizeToString(text, languageIdCodec, tokenizationSupport) {
+    let result = `<div class="monaco-tokenized-source">`;
+    const lines = splitLines(text);
+    let currentState = tokenizationSupport.getInitialState();
+    for (let i = 0, len = lines.length; i < len; i++) {
+        const line = lines[i];
+        if (i > 0) {
+            result += `<br/>`;
+        }
+        const tokenizationResult = tokenizationSupport.tokenizeEncoded(line, true, currentState);
+        LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
+        const lineTokens = new LineTokens(tokenizationResult.tokens, line, languageIdCodec);
+        const viewLineTokens = lineTokens.inflate();
+        let startOffset = 0;
+        for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) {
+            const type = viewLineTokens.getClassName(j);
+            const endIndex = viewLineTokens.getEndOffset(j);
+            result += `<span class="${type}">${escape(line.substring(startOffset, endIndex))}</span>`;
+            startOffset = endIndex;
+        }
+        currentState = tokenizationResult.endState;
+    }
+    result += `</div>`;
+    return result;
+}
+
+export { _tokenizeToString, tokenizeLineToHTML, tokenizeToString };

package/dist/workers/textualHighlightProvider.js

@@ -0,0 +1,78 @@
+import { USUAL_WORD_SEPARATORS } from '../../../common/core/wordHelper.js';
+import { ILanguageFeaturesService } from '../../../common/services/languageFeatures.js';
+import { DocumentHighlightKind } from '../../../common/languages.js';
+import { Disposable } from '../../../../base/common/lifecycle.js';
+import { ResourceMap } from '../../../../base/common/map.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+var __decorate = (undefined && undefined.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (undefined && undefined.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+class TextualDocumentHighlightProvider {
+    constructor() {
+        this.selector = { language: '*' };
+    }
+    provideDocumentHighlights(model, position, token) {
+        const result = [];
+        const word = model.getWordAtPosition({
+            lineNumber: position.lineNumber,
+            column: position.column
+        });
+        if (!word) {
+            return Promise.resolve(result);
+        }
+        if (model.isDisposed()) {
+            return;
+        }
+        const matches = model.findMatches(word.word, true, false, true, USUAL_WORD_SEPARATORS, false);
+        return matches.map(m => ({
+            range: m.range,
+            kind: DocumentHighlightKind.Text
+        }));
+    }
+    provideMultiDocumentHighlights(primaryModel, position, otherModels, token) {
+        const result = new ResourceMap();
+        const word = primaryModel.getWordAtPosition({
+            lineNumber: position.lineNumber,
+            column: position.column
+        });
+        if (!word) {
+            return Promise.resolve(result);
+        }
+        for (const model of [primaryModel, ...otherModels]) {
+            if (model.isDisposed()) {
+                continue;
+            }
+            const matches = model.findMatches(word.word, true, false, true, USUAL_WORD_SEPARATORS, false);
+            const highlights = matches.map(m => ({
+                range: m.range,
+                kind: DocumentHighlightKind.Text
+            }));
+            if (highlights) {
+                result.set(model.uri, highlights);
+            }
+        }
+        return result;
+    }
+}
+let TextualMultiDocumentHighlightFeature = class TextualMultiDocumentHighlightFeature extends Disposable {
+    constructor(languageFeaturesService) {
+        super();
+        this._register(languageFeaturesService.documentHighlightProvider.register('*', new TextualDocumentHighlightProvider()));
+        this._register(languageFeaturesService.multiDocumentHighlightProvider.register('*', new TextualDocumentHighlightProvider()));
+    }
+};
+TextualMultiDocumentHighlightFeature = __decorate([
+    __param(0, ILanguageFeaturesService)
+], TextualMultiDocumentHighlightFeature);
+
+export { TextualMultiDocumentHighlightFeature };