@vscode/chat-lib 0.0.1

package/src/_internal/extension/inlineEdits/common/editRebase.ts

@@ -0,0 +1,268 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { SingleEdits } from '../../../platform/inlineEdits/common/dataTypes/edit';
+import * as errors from '../../../util/common/errors';
+import { ITracer } from '../../../util/common/tracing';
+import { AnnotatedStringEdit, AnnotatedStringReplacement, IEditData, StringEdit, StringReplacement, VoidEditData } from '../../../util/vs/editor/common/core/edits/stringEdit';
+import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
+import { StringText } from '../../../util/vs/editor/common/core/text/abstractText';
+import { DefaultLinesDiffComputer } from '../../../util/vs/editor/common/diff/defaultLinesDiffComputer/defaultLinesDiffComputer';
+import { ILinesDiffComputerOptions } from '../../../util/vs/editor/common/diff/linesDiffComputer';
+
+const TROUBLESHOOT_EDIT_CONSISTENCY = false;
+
+export interface NesRebaseConfigs {
+}
+
+export class EditDataWithIndex implements IEditData<EditDataWithIndex> {
+	constructor(
+		public readonly index: number
+	) { }
+
+	join(data: EditDataWithIndex): EditDataWithIndex | undefined {
+		if (this.index !== data.index) {
+			return undefined;
+		}
+		return this;
+	}
+}
+
+export function tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSince: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs = {}): { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] | 'outsideEditWindow' | 'rebaseFailed' | 'error' | 'inconsistentEdits' {
+	const start = Date.now();
+	try {
+		return _tryRebase(originalDocument, editWindow, originalEdits, detailedEdits, userEditSince, currentDocumentContent, currentSelection, resolution, tracer, nesConfigs);
+	} catch (err) {
+		tracer.trace(`Rebase error: ${errors.toString(err)}`);
+		return 'error';
+	} finally {
+		tracer.trace(`Rebase duration: ${Date.now() - start}ms`);
+	}
+}
+
+function _tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSinceOrig: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs) {
+	if (!checkEditConsistency(originalDocument, userEditSinceOrig, currentDocumentContent, tracer, true)) {
+		return 'inconsistentEdits';
+	}
+	const userEditSince = userEditSinceOrig.removeCommonSuffixAndPrefix(originalDocument);
+	const cursorRange = currentSelection[0];
+	if (editWindow && cursorRange) {
+		const updatedEditWindow = userEditSince.applyToOffsetRangeOrUndefined(editWindow);
+		if (!updatedEditWindow?.containsRange(cursorRange)) {
+			return 'outsideEditWindow';
+		}
+	}
+	if (detailedEdits.length < originalEdits.length) {
+		let intermediateDocument = originalDocument;
+		for (let index = 0; index < detailedEdits.length; index++) {
+			const edit = originalEdits[index];
+			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
+		}
+		for (let index = detailedEdits.length; index < originalEdits.length; index++) {
+			const edit = originalEdits[index];
+			const editData = new EditDataWithIndex(index);
+			detailedEdits[index] = computeDiff(edit.replaceRange.substring(intermediateDocument), edit.newText, edit.replaceRange.start, editData, {
+				ignoreTrimWhitespace: false,
+				computeMoves: false,
+				extendToSubwords: true,
+				maxComputationTimeMs: 500,
+			}) || [new AnnotatedStringReplacement(edit.replaceRange, edit.newText, editData)];
+			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
+		}
+	}
+	const diffedEdit = AnnotatedStringEdit.compose(detailedEdits.map(edits => AnnotatedStringEdit.create(edits)));
+	const rebasedEdit = tryRebaseEdits(originalDocument, diffedEdit, userEditSince, resolution, nesConfigs);
+	if (!rebasedEdit) {
+		return 'rebaseFailed';
+	}
+	const grouped = rebasedEdit.replacements.reduce((acc, item) => {
+		(acc[item.data.index] ||= []).push(item);
+		return acc;
+	}, [] as (AnnotatedStringReplacement<EditDataWithIndex>[] | undefined)[]);
+	const resultEdits: { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] = [];
+	for (let index = 0; index < grouped.length; index++) {
+		const group = grouped[index];
+		if (!group) {
+			continue;
+		}
+		const range = OffsetRange.fromTo(group[0].replaceRange.start, group[group.length - 1].replaceRange.endExclusive);
+		const newText = group.map((edit, i, a) => {
+			if (i > 0) {
+				return currentDocumentContent.substring(a[i - 1].replaceRange.endExclusive, edit.replaceRange.start) + edit.newText;
+			} else {
+				return edit.newText;
+			}
+		}).join('');
+		const resultEdit = StringReplacement.replace(range, newText);
+		if (!resultEdit.removeCommonSuffixAndPrefix(currentDocumentContent).isEmpty) {
+			resultEdits.push({ rebasedEdit: resultEdit, rebasedEditIndex: index });
+		}
+	}
+	if (resolution === 'strict' && resultEdits.length > 0 && new SingleEdits(originalEdits).apply(originalDocument) !== StringEdit.create(resultEdits.map(r => r.rebasedEdit)).apply(currentDocumentContent)) {
+		tracer.trace('Result consistency check failed.');
+		return 'inconsistentEdits';
+	}
+	return resultEdits;
+}
+
+export function checkEditConsistency(original: string, edit: StringEdit, current: string, tracer: ITracer, enabled = TROUBLESHOOT_EDIT_CONSISTENCY) {
+	if (!enabled) {
+		return true;
+	}
+	const consistent = edit.apply(original) === current;
+	if (!consistent) {
+		tracer.trace('Edit consistency check failed.');
+	}
+	return consistent;
+}
+
+export function tryRebaseStringEdits<T extends IEditData<T>>(content: string, ours: StringEdit, base: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs = {}): StringEdit | undefined {
+	return tryRebaseEdits(content, ours.mapData(r => new VoidEditData()), base, resolution, nesConfigs)?.toStringEdit();
+}
+
+function tryRebaseEdits<T extends IEditData<T>>(content: string, ours: AnnotatedStringEdit<T>, baseOrig: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs): AnnotatedStringEdit<T> | undefined {
+	const base = baseOrig.removeCommonSuffixAndPrefix(content);
+
+	const newEdits: AnnotatedStringReplacement<T>[] = [];
+
+	let baseIdx = 0;
+	let ourIdx = 0;
+	let offset = 0;
+
+	while (ourIdx < ours.replacements.length || baseIdx < base.replacements.length) {
+		// take the edit that starts first
+		const baseEdit = base.replacements[baseIdx];
+		const ourEdit = ours.replacements[ourIdx];
+
+		if (!ourEdit) {
+			if (resolution === 'strict') {
+				// baseEdit does not match but interleaves
+				return undefined;
+			}
+			// We processed all our edits
+			break;
+		} else if (!baseEdit) {
+			// no more edits from base
+			newEdits.push(ourEdit.delta(offset));
+			ourIdx++;
+		} else {
+			let ourE = ourEdit;
+			if (!ourE.replaceRange.containsRange(baseEdit.replaceRange)) {
+				// Try to shift our edit to include the base edit.
+				if (ourE.replaceRange.start > baseEdit.replaceRange.start) {
+					// Expand our edit to the left to include the base edit.
+					const added = content.substring(baseEdit.replaceRange.start, ourE.replaceRange.start);
+					const updated = added + ourE.newText;
+					// Remove the same text from the right.
+					if (updated.endsWith(added)) {
+						ourE = new AnnotatedStringReplacement(
+							OffsetRange.fromTo(baseEdit.replaceRange.start, ourE.replaceRange.endExclusive - added.length),
+							updated.substring(0, updated.length - added.length),
+							ourE.data,
+						);
+					}
+				}
+				// Skipping the case where there is another edit for now because we might have to merge with it first.
+				else if (ourIdx === ours.replacements.length - 1 && ourE.replaceRange.endExclusive < baseEdit.replaceRange.endExclusive) {
+					// Expand our edit to the right to include the base edit.
+					const added = content.substring(ourE.replaceRange.endExclusive, baseEdit.replaceRange.endExclusive);
+					const updated = ourE.newText + added;
+					// Remove the same text from the left.
+					if (updated.startsWith(added)) {
+						ourE = new AnnotatedStringReplacement(
+							OffsetRange.fromTo(ourE.replaceRange.start + added.length, baseEdit.replaceRange.endExclusive),
+							updated.substring(added.length),
+							ourE.data,
+						);
+					}
+				}
+			}
+			if (ourE.replaceRange.intersectsOrTouches(baseEdit.replaceRange)) {
+				if (ourE.replaceRange.containsRange(baseEdit.replaceRange) && ourE.newText.length >= baseEdit.newText.length) {
+					let delta = 0;
+					let ourNewTextOffset = 0;
+					let baseE = baseEdit;
+					let previousBaseE: StringReplacement | undefined;
+					while (baseE && ourE.replaceRange.containsRange(baseE.replaceRange)) {
+						ourNewTextOffset = agreementIndexOf(content, ourE, baseE, previousBaseE, ourNewTextOffset, resolution, nesConfigs);
+						if (ourNewTextOffset === -1) {
+							// Conflicting
+							return undefined;
+						}
+						delta += baseE.newText.length - baseE.replaceRange.length;
+						previousBaseE = baseE;
+						baseE = base.replacements[++baseIdx];
+					}
+					newEdits.push(new AnnotatedStringReplacement(
+						new OffsetRange(ourE.replaceRange.start + offset, ourE.replaceRange.endExclusive + offset + delta),
+						ourE.newText,
+						ourE.data,
+					));
+					ourIdx++;
+					offset += delta;
+				} else {
+					// Conflicting
+					return undefined;
+				}
+			} else if (ourEdit.replaceRange.start < baseEdit.replaceRange.start) {
+				// Our edit starts first
+				newEdits.push(new AnnotatedStringReplacement(
+					ourEdit.replaceRange.delta(offset),
+					ourEdit.newText,
+					ourEdit.data,
+				));
+				ourIdx++;
+			} else {
+				if (resolution === 'strict') {
+					// baseEdit does not match but interleaves
+					return undefined;
+				}
+				baseIdx++;
+				offset += baseEdit.newText.length - baseEdit.replaceRange.length;
+			}
+		}
+	}
+
+	return AnnotatedStringEdit.create(newEdits);
+}
+
+export const maxAgreementOffset = 10; // If the user's typing is more than this into the suggestion we consider it a miss.
+export const maxImperfectAgreementLength = 5; // If the user's typing is longer than this and the suggestion is not a perfect match we consider it a miss.
+
+function agreementIndexOf<T extends IEditData<T>>(content: string, ourE: AnnotatedStringReplacement<T>, baseE: StringReplacement, previousBaseE: StringReplacement | undefined, ourNewTextOffset: number, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs) {
+	const minStart = previousBaseE ? previousBaseE.replaceRange.endExclusive : ourE.replaceRange.start;
+	if (minStart < baseE.replaceRange.start) {
+		baseE = new StringReplacement(
+			OffsetRange.fromTo(minStart, baseE.replaceRange.endExclusive),
+			content.substring(minStart, baseE.replaceRange.start) + baseE.newText
+		);
+	}
+	const j = ourE.newText.indexOf(baseE.newText, ourNewTextOffset);
+	if (resolution === 'strict' && j > maxAgreementOffset) {
+		return -1;
+	}
+	if (resolution === 'strict' && j > 0 && baseE.newText.length > maxImperfectAgreementLength) {
+		return -1;
+	}
+	return j !== -1 ? j + baseE.newText.length : -1;
+}
+
+function computeDiff(original: string, modified: string, offset: number, editData: EditDataWithIndex, options: ILinesDiffComputerOptions): AnnotatedStringReplacement<EditDataWithIndex>[] | undefined {
+	const originalLines = original.split(/\r\n|\r|\n/);
+	const modifiedLines = modified.split(/\r\n|\r|\n/);
+	const diffComputer = new DefaultLinesDiffComputer();
+	const result = diffComputer.computeDiff(originalLines, modifiedLines, options);
+	if (result.hitTimeout) {
+		return undefined;
+	}
+
+	const originalText = new StringText(original);
+	const modifiedText = new StringText(modified);
+	return result.changes.map(change => (change.innerChanges || []).map(innerChange => {
+		const range = originalText.getTransformer().getOffsetRange(innerChange.originalRange);
+		const newText = modifiedText.getValueOfRange(innerChange.modifiedRange);
+		return new AnnotatedStringReplacement(range.delta(offset), newText, editData);
+	})).flat();
+}

package/src/_internal/extension/inlineEdits/common/ghNearbyNesProvider.tsx

@@ -0,0 +1,163 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { StatelessNextEditDocument } from '../../../platform/inlineEdits/common/statelessNextEditProvider';
+import { LineEdit } from '../../../util/vs/editor/common/core/edits/lineEdit';
+import { StringEdit } from '../../../util/vs/editor/common/core/edits/stringEdit';
+import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
+
+const N_GRAM_UNDO_RATIO_TO_FILTER_OUT = 0.7;
+
+export function editWouldDeleteWhatWasJustInserted(activeDocument: StatelessNextEditDocument, lineEdit: LineEdit) {
+	let edit = lineEdit.toEdit(activeDocument.documentAfterEdits);
+	// ! important: reduce it to the minimal set of changes
+	edit = edit.normalizeOnSource(activeDocument.documentAfterEdits.value);
+	if (!editIsDeletion(edit)) {
+		return false;
+	}
+	// We are deleting something. Is it what was just inserted?
+	for (let i = activeDocument.recentEdits.edits.length - 1; i >= 0; i--) {
+		const recentEdit = activeDocument.recentEdits.edits[i];
+		const rebaseResult = edit.tryRebase(recentEdit);
+		if (!rebaseResult) {
+			// the edit we want to do cannot be rebased, which indicates that it would interfere with a recent edit
+			return true;
+		}
+		edit = rebaseResult;
+	}
+	return false;
+}
+
+function editIsDeletion(edit: StringEdit): boolean {
+	const deletedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.replaceRange.length, 0);
+	const insertedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.newText.length, 0);
+	return insertedChars === 0 && deletedChars > 0;
+}
+
+/**
+ * Represents information loss/gain (4-grams) via an edit.
+ */
+export class InformationDelta {
+
+	constructor(
+		public readonly inserted: Set<string> = new Set<string>(),
+		public readonly deleted: Set<string> = new Set<string>()
+	) { }
+
+	combine(other: InformationDelta) {
+		return new InformationDelta(setUnion(this.inserted, other.inserted), setUnion(this.deleted, other.deleted));
+	}
+
+	isUndoneBy(other: InformationDelta) {
+		const otherReallyNewInsertions = setMinus(other.inserted, other.deleted);
+		const otherReallyDeleted = setMinus(other.deleted, other.inserted);
+
+		const otherReallyDeletesMyInserts = setIntersectionCount(otherReallyDeleted, this.inserted);
+		const otherReallyInsertsMyDeletes = setIntersectionCount(otherReallyNewInsertions, this.deleted);
+
+		if (otherReallyDeleted.size > 6 && otherReallyDeletesMyInserts / otherReallyDeleted.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
+			return true;
+		}
+
+		if (otherReallyNewInsertions.size > 6 && otherReallyInsertsMyDeletes / otherReallyNewInsertions.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
+			return true;
+		}
+
+		return false;
+	}
+}
+
+export function getInformationDelta(source: string, edit: StringEdit): InformationDelta {
+	const inserted = new Set<string>();
+	const deleted = new Set<string>();
+	const tryAddDeleted = (deletedRange: OffsetRange | undefined) => {
+		if (!deletedRange) {
+			return;
+		}
+		const deletedText = source.substring(deletedRange.start, deletedRange.endExclusive);
+		for (let line of deletedText.split(/\r\n|\r|\n/)) {
+			line = line.trim();
+			for (const piece of to4grams(line)) {
+				deleted.add(piece);
+			}
+		}
+	};
+	const tryAddInserted = (insertedText: string) => {
+		for (let line of insertedText.split(/\r\n|\r|\n/)) {
+			line = line.trim();
+			for (const piece of to4grams(line)) {
+				inserted.add(piece);
+			}
+		}
+	};
+	for (const e of edit.replacements) {
+		const e1 = e.removeCommonPrefix(source).removeCommonSuffix(source);
+		const e2 = e.removeCommonSuffix(source).removeCommonPrefix(source);
+		if (e1.isEmpty) {
+			continue;
+		}
+		tryAddDeleted(e1.replaceRange);
+		tryAddDeleted(e2.replaceRange);
+		tryAddDeleted(e1.replaceRange.intersect(e2.replaceRange));
+
+		// tryAddInserted(e1.newText);
+		// tryAddInserted(e2.newText);
+		// e1 might have a suffix overlap with the prefix of e1
+		tryAddInserted(trimOverlap(e1.newText, e2.newText));
+	}
+	return new InformationDelta(inserted, deleted);
+}
+
+function trimOverlap(stringToEliminateEnd: string, stringToEliminateStart: string): string {
+	const length = Math.min(stringToEliminateEnd.length, stringToEliminateStart.length);
+	for (let trimLength = 0; trimLength < length; trimLength++) {
+		const str1 = stringToEliminateEnd.slice(0, stringToEliminateEnd.length - trimLength);
+		const str2 = stringToEliminateStart.slice(trimLength);
+		if (str1 === str2) {
+			return str1;
+		}
+	}
+	return '';
+}
+
+function to4grams(text: string) {
+	const result: string[] = [];
+	for (let i = 4; i < text.length; i++) {
+		const ngram = text.slice(i - 4, i);
+		result.push(ngram);
+	}
+	return result;
+}
+
+function setUnion(a: Set<string>, b: Set<string>): Set<string> {
+	const result = new Set<string>();
+	for (const el of a) {
+		result.add(el);
+	}
+	for (const el of b) {
+		result.add(el);
+	}
+	return result;
+}
+
+function setMinus(a: Set<string>, b: Set<string>): Set<string> {
+	const result = new Set<string>();
+	for (const el of a) {
+		if (!b.has(el)) {
+			result.add(el);
+		}
+	}
+	return result;
+}
+
+function setIntersectionCount(a: Set<string>, b: Set<string>): number {
+	let result = 0;
+	for (const el of a) {
+		if (b.has(el)) {
+			result++;
+		}
+	}
+	return result;
+}

package/src/_internal/extension/inlineEdits/common/nearbyCursorInlineEditProvider.ts

@@ -0,0 +1,225 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { StatelessNextEditDocument } from '../../../platform/inlineEdits/common/statelessNextEditProvider';
+import { ChoiceLogProbs } from '../../../platform/networking/common/openai';
+import { BugIndicatingError } from '../../../util/vs/base/common/errors';
+import { Range } from '../../../util/vs/editor/common/core/range';
+import { OffsetRange, OffsetRangeSet } from '../../../util/vs/editor/common/core/ranges/offsetRange';
+
+/**
+ * Read the selection from the document, otherwise deduce it from the last edit.
+ */
+export function getOrDeduceSelectionFromLastEdit(activeDoc: StatelessNextEditDocument): Range | null {
+	const origin = new OffsetRange(0, 0);
+	if (activeDoc.lastSelectionInAfterEdit && !activeDoc.lastSelectionInAfterEdit.equals(origin)) {
+		return activeDoc.documentAfterEdits.getTransformer().getRange(activeDoc.lastSelectionInAfterEdit);
+	}
+
+	const selectionRange = deduceSelectionFromLastEdit(activeDoc);
+	return selectionRange;
+}
+
+function deduceSelectionFromLastEdit(activeDoc: StatelessNextEditDocument): Range | null {
+	const mostRecentEdit = activeDoc.recentEdits.edits.at(-1);
+	if (mostRecentEdit === undefined) {
+		return null;
+	}
+
+	const mostRecentSingleEdit = mostRecentEdit.replacements.at(-1);
+	if (mostRecentSingleEdit === undefined) {
+		return null;
+	}
+
+	const offsetRange = mostRecentSingleEdit.replaceRange;
+	const newText = mostRecentSingleEdit.newText;
+	const change = newText.length - offsetRange.length;
+	const newOffset = offsetRange.endExclusive + change;
+
+	const selectionRange = activeDoc.documentAfterEdits.getTransformer().getRange(new OffsetRange(newOffset, newOffset));
+
+	return selectionRange;
+}
+
+type Tokens = Token<number>[];
+
+export class Token<T> {
+	public readonly range: OffsetRange;
+
+	get id(): string {
+		return this.text + '_' + this.range.toString();
+	}
+
+	constructor(public readonly text: string, public readonly value: T, offset: number) {
+		this.range = new OffsetRange(offset, offset + text.length);
+	}
+
+	public equals(other: Token<T>): boolean {
+		return this.range.equals(other.range) && this.text === other.text;
+	}
+
+	public deltaOffset(offset: number): Token<T> {
+		return new Token(this.text, this.value, this.range.start + offset);
+	}
+}
+
+export function clipTokensToRange(tokens: Tokens, range: OffsetRange): Tokens {
+	return tokens.filter(token => range.intersects(token.range));
+}
+
+export function clipTokensToRangeAndAdjustOffsets(tokens: Tokens, range: OffsetRange): Tokens {
+	return clipTokensToRange(tokens, range).map(token => token.deltaOffset(-range.start));
+}
+
+export function removeTokensInRangeAndAdjustOffsets(tokens: Tokens, range: OffsetRange): Tokens {
+	const adjustedTokens: Tokens = [];
+	for (let token of tokens) {
+		// remove tokens inside the range
+		if (range.containsRange(token.range)) {
+			continue;
+		}
+		// adjust the token offset
+		if (token.range.start > range.start) {
+			token = token.deltaOffset(-range.length);
+		}
+
+		adjustedTokens.push(token);
+	}
+
+	return adjustedTokens;
+}
+
+export function getTokensFromLogProbs(logProbs: ChoiceLogProbs, offset: number): Tokens {
+	let acc = offset;
+	return logProbs.content.map(tokenContent => {
+		const token = new Token(tokenContent.token, tokenContent.logprob, acc);
+		acc += token.range.length;
+		return token;
+	});
+}
+
+export class LineWithTokens {
+
+	static stringEquals(a: LineWithTokens, b: LineWithTokens): boolean {
+		return a._text === b._text;
+	}
+
+	static fromText(text: string, tokens: Tokens | undefined): LineWithTokens[] {
+		tokens = tokens ?? [];
+
+		const lines: LineWithTokens[] = [];
+		while (true) {
+			const eolIdxWith = text.indexOf('\r\n');
+			const eolIdxWithout = text.indexOf('\n');
+			const eolIdx = (eolIdxWith === -1 ? eolIdxWithout : (eolIdxWithout === -1 ? eolIdxWith : Math.min(eolIdxWith, eolIdxWithout)));
+			const eol = (eolIdxWith !== -1 ? '\r\n' : (eolIdxWithout === -1 ? undefined : '\n'));
+
+			if (eol === undefined) {
+				lines.push(new LineWithTokens(text, tokens, '\n'));
+				break;
+			}
+
+			const lineLength = eolIdx + eol.length;
+			const line = text.substring(0, eolIdx);
+			const lineTokensWithBoundary = tokens.filter(t => t.range.start < lineLength && t.range.endExclusive > 0);
+			lines.push(new LineWithTokens(line, lineTokensWithBoundary, eol));
+
+			text = text.substring(lineLength);
+			tokens = tokens.map(t => t.deltaOffset(-lineLength)).filter(t => t.range.endExclusive > 0);
+		}
+
+		return lines;
+	}
+
+	get text(): string { return this._text; }
+	get tokens(): Tokens { return this._tokens; }
+	get length(): number { return this._text.length; }
+	get lengthWithEOL(): number { return this._text.length + this._eol.length; }
+	get eol(): '\n' | '\r\n' { return this._eol; }
+
+	constructor(
+		private readonly _text: string,
+		private readonly _tokens: Tokens,
+		private readonly _eol: '\n' | '\r\n'
+	) { }
+
+	trim() {
+		return this.trimStart().trimEnd();
+	}
+
+	trimStart() {
+		const lineStartTrimmed = this._text.trimStart();
+		const trimmedLength = this._text.length - lineStartTrimmed.length;
+		const tokensUpdated = this._tokens.map(t => t.deltaOffset(-trimmedLength)).filter(t => t.range.endExclusive > 0);
+		return new LineWithTokens(lineStartTrimmed, tokensUpdated, this._eol);
+	}
+
+	trimEnd() {
+		const lineEndTrimmed = this._text.trimEnd();
+		const tokensUpdated = this._tokens.filter(t => t.range.start < lineEndTrimmed.length);
+		return new LineWithTokens(lineEndTrimmed, tokensUpdated, this._eol);
+	}
+
+	substring(start: number, end: number): LineWithTokens {
+		const lineSubstring = this._text.substring(start, end);
+		const tokensUpdated = this._tokens.map(t => t.deltaOffset(-start)).filter(t => t.range.endExclusive > 0 && t.range.start < lineSubstring.length);
+		return new LineWithTokens(lineSubstring, tokensUpdated, this._eol);
+	}
+
+	stringEquals(other: LineWithTokens): boolean {
+		return LineWithTokens.stringEquals(this, other);
+	}
+
+	equals(other: LineWithTokens): boolean {
+		return this._text === other.text
+			&& this._tokens.length === other.tokens.length
+			&& this._tokens.every((t, i) => t.equals(other.tokens[i]));
+	}
+
+	dropTokens(tokens: Tokens): LineWithTokens {
+		return new LineWithTokens(this._text, this._tokens.filter(t => !tokens.some(token => t.equals(token))), this._eol);
+	}
+
+	findTokens(fn: (token: Token<number>) => boolean): Token<number>[] {
+		return this._tokens.filter(fn);
+	}
+}
+
+export function getTokensFromLinesWithTokens(lines: LineWithTokens[]): Tokens {
+	let offset = 0;
+
+	const tokens: Tokens = [];
+	for (const line of lines) {
+		const textLine = line.text + line.eol;
+		tokens.push(...line.tokens.map(t => t.deltaOffset(offset)));
+		offset += textLine.length;
+	}
+
+	const tokensDeduplicated: Tokens = [];
+	const tokensSeen = new Set<string>();
+	for (const token of tokens) {
+		if (!tokensSeen.has(token.id)) {
+			tokensSeen.add(token.id);
+			tokensDeduplicated.push(token);
+		}
+	}
+
+	return tokensDeduplicated;
+}
+
+export function mergeOffsetRangesAtDistance(ranges: OffsetRange[], distance: number): OffsetRange[] {
+	if (distance < 0) {
+		throw new BugIndicatingError('Distance must be positive');
+	}
+
+	const rangesGrown = ranges.map(r => new OffsetRange(r.start - distance, r.endExclusive + distance));
+
+	const set = new OffsetRangeSet();
+	for (const range of rangesGrown) {
+		set.addRange(range);
+	}
+
+	return set.ranges.map(r => new OffsetRange(r.start + distance, r.endExclusive - distance));
+}