monaco-editor11 1.0.7 → 1.0.9

package/dist/workers/editor/common/columnRange.js

@@ -0,0 +1,35 @@
+import { BugIndicatingError } from '../../../../base/common/errors.js';
+import { OffsetRange } from './offsetRange.js';
+import { Range } from '../range.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+/**
+ * Represents a 1-based range of columns.
+ * Use {@lik OffsetRange} to represent a 0-based range.
+*/
+class ColumnRange {
+    constructor(
+    /** 1-based */
+    startColumn, endColumnExclusive) {
+        this.startColumn = startColumn;
+        this.endColumnExclusive = endColumnExclusive;
+        if (startColumn > endColumnExclusive) {
+            throw new BugIndicatingError(`startColumn ${startColumn} cannot be after endColumnExclusive ${endColumnExclusive}`);
+        }
+    }
+    toRange(lineNumber) {
+        return new Range(lineNumber, this.startColumn, lineNumber, this.endColumnExclusive);
+    }
+    equals(other) {
+        return this.startColumn === other.startColumn
+            && this.endColumnExclusive === other.endColumnExclusive;
+    }
+    toZeroBasedOffsetRange() {
+        return new OffsetRange(this.startColumn - 1, this.endColumnExclusive - 1);
+    }
+}
+
+export { ColumnRange };

package/dist/workers/editor/common/combineTextEditInfos.js

@@ -0,0 +1,124 @@
+import { ArrayQueue } from '../../../../../base/common/arrays.js';
+import { TextEditInfo } from './beforeEditPositionMapper.js';
+import { sumLengths, lengthAdd, lengthDiffNonNegative, lengthIsZero, lengthEquals, lengthZero, lengthToObj } from './length.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+function combineTextEditInfos(textEditInfoFirst, textEditInfoSecond) {
+    if (textEditInfoFirst.length === 0) {
+        return textEditInfoSecond;
+    }
+    if (textEditInfoSecond.length === 0) {
+        return textEditInfoFirst;
+    }
+    // s0: State before any edits
+    const s0ToS1Map = new ArrayQueue(toLengthMapping(textEditInfoFirst));
+    // s1: State after first edit, but before second edit
+    const s1ToS2Map = toLengthMapping(textEditInfoSecond);
+    s1ToS2Map.push({ modified: false, lengthBefore: undefined, lengthAfter: undefined }); // Copy everything from old to new
+    // s2: State after both edits
+    let curItem = s0ToS1Map.dequeue();
+    /**
+     * @param s1Length Use undefined for length "infinity"
+     */
+    function nextS0ToS1MapWithS1LengthOf(s1Length) {
+        if (s1Length === undefined) {
+            const arr = s0ToS1Map.takeWhile(v => true) || [];
+            if (curItem) {
+                arr.unshift(curItem);
+            }
+            return arr;
+        }
+        const result = [];
+        while (curItem && !lengthIsZero(s1Length)) {
+            const [item, remainingItem] = curItem.splitAt(s1Length);
+            result.push(item);
+            s1Length = lengthDiffNonNegative(item.lengthAfter, s1Length);
+            curItem = remainingItem ?? s0ToS1Map.dequeue();
+        }
+        if (!lengthIsZero(s1Length)) {
+            result.push(new LengthMapping(false, s1Length, s1Length));
+        }
+        return result;
+    }
+    const result = [];
+    function pushEdit(startOffset, endOffset, newLength) {
+        if (result.length > 0 && lengthEquals(result[result.length - 1].endOffset, startOffset)) {
+            const lastResult = result[result.length - 1];
+            result[result.length - 1] = new TextEditInfo(lastResult.startOffset, endOffset, lengthAdd(lastResult.newLength, newLength));
+        }
+        else {
+            result.push({ startOffset, endOffset, newLength });
+        }
+    }
+    let s0offset = lengthZero;
+    for (const s1ToS2 of s1ToS2Map) {
+        const s0ToS1Map = nextS0ToS1MapWithS1LengthOf(s1ToS2.lengthBefore);
+        if (s1ToS2.modified) {
+            const s0Length = sumLengths(s0ToS1Map, s => s.lengthBefore);
+            const s0EndOffset = lengthAdd(s0offset, s0Length);
+            pushEdit(s0offset, s0EndOffset, s1ToS2.lengthAfter);
+            s0offset = s0EndOffset;
+        }
+        else {
+            for (const s1 of s0ToS1Map) {
+                const s0startOffset = s0offset;
+                s0offset = lengthAdd(s0offset, s1.lengthBefore);
+                if (s1.modified) {
+                    pushEdit(s0startOffset, s0offset, s1.lengthAfter);
+                }
+            }
+        }
+    }
+    return result;
+}
+class LengthMapping {
+    constructor(
+    /**
+     * If false, length before and length after equal.
+     */
+    modified, lengthBefore, lengthAfter) {
+        this.modified = modified;
+        this.lengthBefore = lengthBefore;
+        this.lengthAfter = lengthAfter;
+    }
+    splitAt(lengthAfter) {
+        const remainingLengthAfter = lengthDiffNonNegative(lengthAfter, this.lengthAfter);
+        if (lengthEquals(remainingLengthAfter, lengthZero)) {
+            return [this, undefined];
+        }
+        else if (this.modified) {
+            return [
+                new LengthMapping(this.modified, this.lengthBefore, lengthAfter),
+                new LengthMapping(this.modified, lengthZero, remainingLengthAfter)
+            ];
+        }
+        else {
+            return [
+                new LengthMapping(this.modified, lengthAfter, lengthAfter),
+                new LengthMapping(this.modified, remainingLengthAfter, remainingLengthAfter)
+            ];
+        }
+    }
+    toString() {
+        return `${this.modified ? 'M' : 'U'}:${lengthToObj(this.lengthBefore)} -> ${lengthToObj(this.lengthAfter)}`;
+    }
+}
+function toLengthMapping(textEditInfos) {
+    const result = [];
+    let lastOffset = lengthZero;
+    for (const textEditInfo of textEditInfos) {
+        const spaceLength = lengthDiffNonNegative(lastOffset, textEditInfo.startOffset);
+        if (!lengthIsZero(spaceLength)) {
+            result.push(new LengthMapping(false, spaceLength, spaceLength));
+        }
+        const lengthBefore = lengthDiffNonNegative(textEditInfo.startOffset, textEditInfo.endOffset);
+        result.push(new LengthMapping(true, lengthBefore, textEditInfo.newLength));
+        lastOffset = textEditInfo.endOffset;
+    }
+    return result;
+}
+
+export { combineTextEditInfos };

package/dist/workers/editor/common/common.js

@@ -0,0 +1,20 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+class RateLimiter {
+    constructor(timesPerSecond = 5) {
+        this.timesPerSecond = timesPerSecond;
+        this._lastRun = 0;
+        this._minimumTimeBetweenRuns = 1000 / timesPerSecond;
+    }
+    runIfNotLimited(callback) {
+        const now = Date.now();
+        if (now - this._lastRun >= this._minimumTimeBetweenRuns) {
+            this._lastRun = now;
+            callback();
+        }
+    }
+}
+
+export { RateLimiter };

package/dist/workers/editor/common/computeMovedLines.js

@@ -0,0 +1,249 @@
+import { SequenceDiff } from './algorithms/diffAlgorithm.js';
+import { LineRangeMapping } from '../rangeMapping.js';
+import { pushMany, compareBy, numberComparator, reverseOrder } from '../../../../base/common/arrays.js';
+import { MonotonousArray, findLastMonotonous } from '../../../../base/common/arraysFind.js';
+import { SetMap } from '../../../../base/common/map.js';
+import { LineRange, LineRangeSet } from '../../core/ranges/lineRange.js';
+import { LinesSliceCharSequence } from './linesSliceCharSequence.js';
+import { LineRangeFragment, isSpace } from './utils.js';
+import { MyersDiffAlgorithm } from './algorithms/myersDiffAlgorithm.js';
+import { Range } from '../../core/range.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+function computeMovedLines(changes, originalLines, modifiedLines, hashedOriginalLines, hashedModifiedLines, timeout) {
+    let { moves, excludedChanges } = computeMovesFromSimpleDeletionsToSimpleInsertions(changes, originalLines, modifiedLines, timeout);
+    if (!timeout.isValid()) {
+        return [];
+    }
+    const filteredChanges = changes.filter(c => !excludedChanges.has(c));
+    const unchangedMoves = computeUnchangedMoves(filteredChanges, hashedOriginalLines, hashedModifiedLines, originalLines, modifiedLines, timeout);
+    pushMany(moves, unchangedMoves);
+    moves = joinCloseConsecutiveMoves(moves);
+    // Ignore too short moves
+    moves = moves.filter(current => {
+        const lines = current.original.toOffsetRange().slice(originalLines).map(l => l.trim());
+        const originalText = lines.join('\n');
+        return originalText.length >= 15 && countWhere(lines, l => l.length >= 2) >= 2;
+    });
+    moves = removeMovesInSameDiff(changes, moves);
+    return moves;
+}
+function countWhere(arr, predicate) {
+    let count = 0;
+    for (const t of arr) {
+        if (predicate(t)) {
+            count++;
+        }
+    }
+    return count;
+}
+function computeMovesFromSimpleDeletionsToSimpleInsertions(changes, originalLines, modifiedLines, timeout) {
+    const moves = [];
+    const deletions = changes
+        .filter(c => c.modified.isEmpty && c.original.length >= 3)
+        .map(d => new LineRangeFragment(d.original, originalLines, d));
+    const insertions = new Set(changes
+        .filter(c => c.original.isEmpty && c.modified.length >= 3)
+        .map(d => new LineRangeFragment(d.modified, modifiedLines, d)));
+    const excludedChanges = new Set();
+    for (const deletion of deletions) {
+        let highestSimilarity = -1;
+        let best;
+        for (const insertion of insertions) {
+            const similarity = deletion.computeSimilarity(insertion);
+            if (similarity > highestSimilarity) {
+                highestSimilarity = similarity;
+                best = insertion;
+            }
+        }
+        if (highestSimilarity > 0.90 && best) {
+            insertions.delete(best);
+            moves.push(new LineRangeMapping(deletion.range, best.range));
+            excludedChanges.add(deletion.source);
+            excludedChanges.add(best.source);
+        }
+        if (!timeout.isValid()) {
+            return { moves, excludedChanges };
+        }
+    }
+    return { moves, excludedChanges };
+}
+function computeUnchangedMoves(changes, hashedOriginalLines, hashedModifiedLines, originalLines, modifiedLines, timeout) {
+    const moves = [];
+    const original3LineHashes = new SetMap();
+    for (const change of changes) {
+        for (let i = change.original.startLineNumber; i < change.original.endLineNumberExclusive - 2; i++) {
+            const key = `${hashedOriginalLines[i - 1]}:${hashedOriginalLines[i + 1 - 1]}:${hashedOriginalLines[i + 2 - 1]}`;
+            original3LineHashes.add(key, { range: new LineRange(i, i + 3) });
+        }
+    }
+    const possibleMappings = [];
+    changes.sort(compareBy(c => c.modified.startLineNumber, numberComparator));
+    for (const change of changes) {
+        let lastMappings = [];
+        for (let i = change.modified.startLineNumber; i < change.modified.endLineNumberExclusive - 2; i++) {
+            const key = `${hashedModifiedLines[i - 1]}:${hashedModifiedLines[i + 1 - 1]}:${hashedModifiedLines[i + 2 - 1]}`;
+            const currentModifiedRange = new LineRange(i, i + 3);
+            const nextMappings = [];
+            original3LineHashes.forEach(key, ({ range }) => {
+                for (const lastMapping of lastMappings) {
+                    // does this match extend some last match?
+                    if (lastMapping.originalLineRange.endLineNumberExclusive + 1 === range.endLineNumberExclusive &&
+                        lastMapping.modifiedLineRange.endLineNumberExclusive + 1 === currentModifiedRange.endLineNumberExclusive) {
+                        lastMapping.originalLineRange = new LineRange(lastMapping.originalLineRange.startLineNumber, range.endLineNumberExclusive);
+                        lastMapping.modifiedLineRange = new LineRange(lastMapping.modifiedLineRange.startLineNumber, currentModifiedRange.endLineNumberExclusive);
+                        nextMappings.push(lastMapping);
+                        return;
+                    }
+                }
+                const mapping = {
+                    modifiedLineRange: currentModifiedRange,
+                    originalLineRange: range,
+                };
+                possibleMappings.push(mapping);
+                nextMappings.push(mapping);
+            });
+            lastMappings = nextMappings;
+        }
+        if (!timeout.isValid()) {
+            return [];
+        }
+    }
+    possibleMappings.sort(reverseOrder(compareBy(m => m.modifiedLineRange.length, numberComparator)));
+    const modifiedSet = new LineRangeSet();
+    const originalSet = new LineRangeSet();
+    for (const mapping of possibleMappings) {
+        const diffOrigToMod = mapping.modifiedLineRange.startLineNumber - mapping.originalLineRange.startLineNumber;
+        const modifiedSections = modifiedSet.subtractFrom(mapping.modifiedLineRange);
+        const originalTranslatedSections = originalSet.subtractFrom(mapping.originalLineRange).getWithDelta(diffOrigToMod);
+        const modifiedIntersectedSections = modifiedSections.getIntersection(originalTranslatedSections);
+        for (const s of modifiedIntersectedSections.ranges) {
+            if (s.length < 3) {
+                continue;
+            }
+            const modifiedLineRange = s;
+            const originalLineRange = s.delta(-diffOrigToMod);
+            moves.push(new LineRangeMapping(originalLineRange, modifiedLineRange));
+            modifiedSet.addRange(modifiedLineRange);
+            originalSet.addRange(originalLineRange);
+        }
+    }
+    moves.sort(compareBy(m => m.original.startLineNumber, numberComparator));
+    const monotonousChanges = new MonotonousArray(changes);
+    for (let i = 0; i < moves.length; i++) {
+        const move = moves[i];
+        const firstTouchingChangeOrig = monotonousChanges.findLastMonotonous(c => c.original.startLineNumber <= move.original.startLineNumber);
+        const firstTouchingChangeMod = findLastMonotonous(changes, c => c.modified.startLineNumber <= move.modified.startLineNumber);
+        const linesAbove = Math.max(move.original.startLineNumber - firstTouchingChangeOrig.original.startLineNumber, move.modified.startLineNumber - firstTouchingChangeMod.modified.startLineNumber);
+        const lastTouchingChangeOrig = monotonousChanges.findLastMonotonous(c => c.original.startLineNumber < move.original.endLineNumberExclusive);
+        const lastTouchingChangeMod = findLastMonotonous(changes, c => c.modified.startLineNumber < move.modified.endLineNumberExclusive);
+        const linesBelow = Math.max(lastTouchingChangeOrig.original.endLineNumberExclusive - move.original.endLineNumberExclusive, lastTouchingChangeMod.modified.endLineNumberExclusive - move.modified.endLineNumberExclusive);
+        let extendToTop;
+        for (extendToTop = 0; extendToTop < linesAbove; extendToTop++) {
+            const origLine = move.original.startLineNumber - extendToTop - 1;
+            const modLine = move.modified.startLineNumber - extendToTop - 1;
+            if (origLine > originalLines.length || modLine > modifiedLines.length) {
+                break;
+            }
+            if (modifiedSet.contains(modLine) || originalSet.contains(origLine)) {
+                break;
+            }
+            if (!areLinesSimilar(originalLines[origLine - 1], modifiedLines[modLine - 1], timeout)) {
+                break;
+            }
+        }
+        if (extendToTop > 0) {
+            originalSet.addRange(new LineRange(move.original.startLineNumber - extendToTop, move.original.startLineNumber));
+            modifiedSet.addRange(new LineRange(move.modified.startLineNumber - extendToTop, move.modified.startLineNumber));
+        }
+        let extendToBottom;
+        for (extendToBottom = 0; extendToBottom < linesBelow; extendToBottom++) {
+            const origLine = move.original.endLineNumberExclusive + extendToBottom;
+            const modLine = move.modified.endLineNumberExclusive + extendToBottom;
+            if (origLine > originalLines.length || modLine > modifiedLines.length) {
+                break;
+            }
+            if (modifiedSet.contains(modLine) || originalSet.contains(origLine)) {
+                break;
+            }
+            if (!areLinesSimilar(originalLines[origLine - 1], modifiedLines[modLine - 1], timeout)) {
+                break;
+            }
+        }
+        if (extendToBottom > 0) {
+            originalSet.addRange(new LineRange(move.original.endLineNumberExclusive, move.original.endLineNumberExclusive + extendToBottom));
+            modifiedSet.addRange(new LineRange(move.modified.endLineNumberExclusive, move.modified.endLineNumberExclusive + extendToBottom));
+        }
+        if (extendToTop > 0 || extendToBottom > 0) {
+            moves[i] = new LineRangeMapping(new LineRange(move.original.startLineNumber - extendToTop, move.original.endLineNumberExclusive + extendToBottom), new LineRange(move.modified.startLineNumber - extendToTop, move.modified.endLineNumberExclusive + extendToBottom));
+        }
+    }
+    return moves;
+}
+function areLinesSimilar(line1, line2, timeout) {
+    if (line1.trim() === line2.trim()) {
+        return true;
+    }
+    if (line1.length > 300 && line2.length > 300) {
+        return false;
+    }
+    const myersDiffingAlgorithm = new MyersDiffAlgorithm();
+    const result = myersDiffingAlgorithm.compute(new LinesSliceCharSequence([line1], new Range(1, 1, 1, line1.length), false), new LinesSliceCharSequence([line2], new Range(1, 1, 1, line2.length), false), timeout);
+    let commonNonSpaceCharCount = 0;
+    const inverted = SequenceDiff.invert(result.diffs, line1.length);
+    for (const seq of inverted) {
+        seq.seq1Range.forEach(idx => {
+            if (!isSpace(line1.charCodeAt(idx))) {
+                commonNonSpaceCharCount++;
+            }
+        });
+    }
+    function countNonWsChars(str) {
+        let count = 0;
+        for (let i = 0; i < line1.length; i++) {
+            if (!isSpace(str.charCodeAt(i))) {
+                count++;
+            }
+        }
+        return count;
+    }
+    const longerLineLength = countNonWsChars(line1.length > line2.length ? line1 : line2);
+    const r = commonNonSpaceCharCount / longerLineLength > 0.6 && longerLineLength > 10;
+    return r;
+}
+function joinCloseConsecutiveMoves(moves) {
+    if (moves.length === 0) {
+        return moves;
+    }
+    moves.sort(compareBy(m => m.original.startLineNumber, numberComparator));
+    const result = [moves[0]];
+    for (let i = 1; i < moves.length; i++) {
+        const last = result[result.length - 1];
+        const current = moves[i];
+        const originalDist = current.original.startLineNumber - last.original.endLineNumberExclusive;
+        const modifiedDist = current.modified.startLineNumber - last.modified.endLineNumberExclusive;
+        const currentMoveAfterLast = originalDist >= 0 && modifiedDist >= 0;
+        if (currentMoveAfterLast && originalDist + modifiedDist <= 2) {
+            result[result.length - 1] = last.join(current);
+            continue;
+        }
+        result.push(current);
+    }
+    return result;
+}
+function removeMovesInSameDiff(changes, moves) {
+    const changesMonotonous = new MonotonousArray(changes);
+    moves = moves.filter(m => {
+        const diffBeforeEndOfMoveOriginal = changesMonotonous.findLastMonotonous(c => c.original.startLineNumber < m.original.endLineNumberExclusive)
+            || new LineRangeMapping(new LineRange(1, 1), new LineRange(1, 1));
+        const diffBeforeEndOfMoveModified = findLastMonotonous(changes, c => c.modified.startLineNumber < m.modified.endLineNumberExclusive);
+        const differentDiffs = diffBeforeEndOfMoveOriginal !== diffBeforeEndOfMoveModified;
+        return differentDiffs;
+    });
+    return moves;
+}
+
+export { computeMovedLines };

package/dist/workers/editor/common/concat23Trees.js

@@ -0,0 +1,192 @@
+import { ListAstNode } from './ast.js';
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+/**
+ * Concatenates a list of (2,3) AstNode's into a single (2,3) AstNode.
+ * This mutates the items of the input array!
+ * If all items have the same height, this method has runtime O(items.length).
+ * Otherwise, it has runtime O(items.length * max(log(items.length), items.max(i => i.height))).
+*/
+function concat23Trees(items) {
+    if (items.length === 0) {
+        return null;
+    }
+    if (items.length === 1) {
+        return items[0];
+    }
+    let i = 0;
+    /**
+     * Reads nodes of same height and concatenates them to a single node.
+    */
+    function readNode() {
+        if (i >= items.length) {
+            return null;
+        }
+        const start = i;
+        const height = items[start].listHeight;
+        i++;
+        while (i < items.length && items[i].listHeight === height) {
+            i++;
+        }
+        if (i - start >= 2) {
+            return concat23TreesOfSameHeight(start === 0 && i === items.length ? items : items.slice(start, i), false);
+        }
+        else {
+            return items[start];
+        }
+    }
+    // The items might not have the same height.
+    // We merge all items by using a binary concat operator.
+    let first = readNode(); // There must be a first item
+    let second = readNode();
+    if (!second) {
+        return first;
+    }
+    for (let item = readNode(); item; item = readNode()) {
+        // Prefer concatenating smaller trees, as the runtime of concat depends on the tree height.
+        if (heightDiff(first, second) <= heightDiff(second, item)) {
+            first = concat(first, second);
+            second = item;
+        }
+        else {
+            second = concat(second, item);
+        }
+    }
+    const result = concat(first, second);
+    return result;
+}
+function concat23TreesOfSameHeight(items, createImmutableLists = false) {
+    if (items.length === 0) {
+        return null;
+    }
+    if (items.length === 1) {
+        return items[0];
+    }
+    let length = items.length;
+    // All trees have same height, just create parent nodes.
+    while (length > 3) {
+        const newLength = length >> 1;
+        for (let i = 0; i < newLength; i++) {
+            const j = i << 1;
+            items[i] = ListAstNode.create23(items[j], items[j + 1], j + 3 === length ? items[j + 2] : null, createImmutableLists);
+        }
+        length = newLength;
+    }
+    return ListAstNode.create23(items[0], items[1], length >= 3 ? items[2] : null, createImmutableLists);
+}
+function heightDiff(node1, node2) {
+    return Math.abs(node1.listHeight - node2.listHeight);
+}
+function concat(node1, node2) {
+    if (node1.listHeight === node2.listHeight) {
+        return ListAstNode.create23(node1, node2, null, false);
+    }
+    else if (node1.listHeight > node2.listHeight) {
+        // node1 is the tree we want to insert into
+        return append(node1, node2);
+    }
+    else {
+        return prepend(node2, node1);
+    }
+}
+/**
+ * Appends the given node to the end of this (2,3) tree.
+ * Returns the new root.
+*/
+function append(list, nodeToAppend) {
+    list = list.toMutable();
+    let curNode = list;
+    const parents = [];
+    let nodeToAppendOfCorrectHeight;
+    while (true) {
+        // assert nodeToInsert.listHeight <= curNode.listHeight
+        if (nodeToAppend.listHeight === curNode.listHeight) {
+            nodeToAppendOfCorrectHeight = nodeToAppend;
+            break;
+        }
+        // assert 0 <= nodeToInsert.listHeight < curNode.listHeight
+        if (curNode.kind !== 4 /* AstNodeKind.List */) {
+            throw new Error('unexpected');
+        }
+        parents.push(curNode);
+        // assert 2 <= curNode.childrenLength <= 3
+        curNode = curNode.makeLastElementMutable();
+    }
+    // assert nodeToAppendOfCorrectHeight!.listHeight === curNode.listHeight
+    for (let i = parents.length - 1; i >= 0; i--) {
+        const parent = parents[i];
+        if (nodeToAppendOfCorrectHeight) {
+            // Can we take the element?
+            if (parent.childrenLength >= 3) {
+                // assert parent.childrenLength === 3 && parent.listHeight === nodeToAppendOfCorrectHeight.listHeight + 1
+                // we need to split to maintain (2,3)-tree property.
+                // Send the third element + the new element to the parent.
+                nodeToAppendOfCorrectHeight = ListAstNode.create23(parent.unappendChild(), nodeToAppendOfCorrectHeight, null, false);
+            }
+            else {
+                parent.appendChildOfSameHeight(nodeToAppendOfCorrectHeight);
+                nodeToAppendOfCorrectHeight = undefined;
+            }
+        }
+        else {
+            parent.handleChildrenChanged();
+        }
+    }
+    if (nodeToAppendOfCorrectHeight) {
+        return ListAstNode.create23(list, nodeToAppendOfCorrectHeight, null, false);
+    }
+    else {
+        return list;
+    }
+}
+/**
+ * Prepends the given node to the end of this (2,3) tree.
+ * Returns the new root.
+*/
+function prepend(list, nodeToAppend) {
+    list = list.toMutable();
+    let curNode = list;
+    const parents = [];
+    // assert nodeToInsert.listHeight <= curNode.listHeight
+    while (nodeToAppend.listHeight !== curNode.listHeight) {
+        // assert 0 <= nodeToInsert.listHeight < curNode.listHeight
+        if (curNode.kind !== 4 /* AstNodeKind.List */) {
+            throw new Error('unexpected');
+        }
+        parents.push(curNode);
+        // assert 2 <= curNode.childrenFast.length <= 3
+        curNode = curNode.makeFirstElementMutable();
+    }
+    let nodeToPrependOfCorrectHeight = nodeToAppend;
+    // assert nodeToAppendOfCorrectHeight!.listHeight === curNode.listHeight
+    for (let i = parents.length - 1; i >= 0; i--) {
+        const parent = parents[i];
+        if (nodeToPrependOfCorrectHeight) {
+            // Can we take the element?
+            if (parent.childrenLength >= 3) {
+                // assert parent.childrenLength === 3 && parent.listHeight === nodeToAppendOfCorrectHeight.listHeight + 1
+                // we need to split to maintain (2,3)-tree property.
+                // Send the third element + the new element to the parent.
+                nodeToPrependOfCorrectHeight = ListAstNode.create23(nodeToPrependOfCorrectHeight, parent.unprependChild(), null, false);
+            }
+            else {
+                parent.prependChildOfSameHeight(nodeToPrependOfCorrectHeight);
+                nodeToPrependOfCorrectHeight = undefined;
+            }
+        }
+        else {
+            parent.handleChildrenChanged();
+        }
+    }
+    if (nodeToPrependOfCorrectHeight) {
+        return ListAstNode.create23(nodeToPrependOfCorrectHeight, list, null, false);
+    }
+    else {
+        return list;
+    }
+}
+
+export { concat23Trees, concat23TreesOfSameHeight };

package/dist/workers/editor/common/contiguousMultilineTokens.js

@@ -0,0 +1,32 @@
+/**
+ * Represents contiguous tokens over a contiguous range of lines.
+ */
+class ContiguousMultilineTokens {
+    /**
+     * (Inclusive) start line number for these tokens.
+     */
+    get startLineNumber() {
+        return this._startLineNumber;
+    }
+    /**
+     * (Inclusive) end line number for these tokens.
+     */
+    get endLineNumber() {
+        return this._startLineNumber + this._tokens.length - 1;
+    }
+    constructor(startLineNumber, tokens) {
+        this._startLineNumber = startLineNumber;
+        this._tokens = tokens;
+    }
+    /**
+     * @see {@link _tokens}
+     */
+    getLineTokens(lineNumber) {
+        return this._tokens[lineNumber - this._startLineNumber];
+    }
+    appendLineTokens(lineTokens) {
+        this._tokens.push(lineTokens);
+    }
+}
+
+export { ContiguousMultilineTokens };

package/dist/workers/editor/common/contiguousMultilineTokensBuilder.js

@@ -0,0 +1,23 @@
+import { ContiguousMultilineTokens } from './contiguousMultilineTokens.js';
+
+class ContiguousMultilineTokensBuilder {
+    constructor() {
+        this._tokens = [];
+    }
+    add(lineNumber, lineTokens) {
+        if (this._tokens.length > 0) {
+            const last = this._tokens[this._tokens.length - 1];
+            if (last.endLineNumber + 1 === lineNumber) {
+                // append
+                last.appendLineTokens(lineTokens);
+                return;
+            }
+        }
+        this._tokens.push(new ContiguousMultilineTokens(lineNumber, [lineTokens]));
+    }
+    finalize() {
+        return this._tokens;
+    }
+}
+
+export { ContiguousMultilineTokensBuilder };