@vscode/chat-lib 0.0.1 → 0.0.3

package/src/_internal/extension/inlineEdits/common/editRebase.ts

@@ -1,268 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { SingleEdits } from '../../../platform/inlineEdits/common/dataTypes/edit';
-import * as errors from '../../../util/common/errors';
-import { ITracer } from '../../../util/common/tracing';
-import { AnnotatedStringEdit, AnnotatedStringReplacement, IEditData, StringEdit, StringReplacement, VoidEditData } from '../../../util/vs/editor/common/core/edits/stringEdit';
-import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
-import { StringText } from '../../../util/vs/editor/common/core/text/abstractText';
-import { DefaultLinesDiffComputer } from '../../../util/vs/editor/common/diff/defaultLinesDiffComputer/defaultLinesDiffComputer';
-import { ILinesDiffComputerOptions } from '../../../util/vs/editor/common/diff/linesDiffComputer';
-
-const TROUBLESHOOT_EDIT_CONSISTENCY = false;
-
-export interface NesRebaseConfigs {
-}
-
-export class EditDataWithIndex implements IEditData<EditDataWithIndex> {
-	constructor(
-		public readonly index: number
-	) { }
-
-	join(data: EditDataWithIndex): EditDataWithIndex | undefined {
-		if (this.index !== data.index) {
-			return undefined;
-		}
-		return this;
-	}
-}
-
-export function tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSince: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs = {}): { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] | 'outsideEditWindow' | 'rebaseFailed' | 'error' | 'inconsistentEdits' {
-	const start = Date.now();
-	try {
-		return _tryRebase(originalDocument, editWindow, originalEdits, detailedEdits, userEditSince, currentDocumentContent, currentSelection, resolution, tracer, nesConfigs);
-	} catch (err) {
-		tracer.trace(`Rebase error: ${errors.toString(err)}`);
-		return 'error';
-	} finally {
-		tracer.trace(`Rebase duration: ${Date.now() - start}ms`);
-	}
-}
-
-function _tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSinceOrig: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs) {
-	if (!checkEditConsistency(originalDocument, userEditSinceOrig, currentDocumentContent, tracer, true)) {
-		return 'inconsistentEdits';
-	}
-	const userEditSince = userEditSinceOrig.removeCommonSuffixAndPrefix(originalDocument);
-	const cursorRange = currentSelection[0];
-	if (editWindow && cursorRange) {
-		const updatedEditWindow = userEditSince.applyToOffsetRangeOrUndefined(editWindow);
-		if (!updatedEditWindow?.containsRange(cursorRange)) {
-			return 'outsideEditWindow';
-		}
-	}
-	if (detailedEdits.length < originalEdits.length) {
-		let intermediateDocument = originalDocument;
-		for (let index = 0; index < detailedEdits.length; index++) {
-			const edit = originalEdits[index];
-			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
-		}
-		for (let index = detailedEdits.length; index < originalEdits.length; index++) {
-			const edit = originalEdits[index];
-			const editData = new EditDataWithIndex(index);
-			detailedEdits[index] = computeDiff(edit.replaceRange.substring(intermediateDocument), edit.newText, edit.replaceRange.start, editData, {
-				ignoreTrimWhitespace: false,
-				computeMoves: false,
-				extendToSubwords: true,
-				maxComputationTimeMs: 500,
-			}) || [new AnnotatedStringReplacement(edit.replaceRange, edit.newText, editData)];
-			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
-		}
-	}
-	const diffedEdit = AnnotatedStringEdit.compose(detailedEdits.map(edits => AnnotatedStringEdit.create(edits)));
-	const rebasedEdit = tryRebaseEdits(originalDocument, diffedEdit, userEditSince, resolution, nesConfigs);
-	if (!rebasedEdit) {
-		return 'rebaseFailed';
-	}
-	const grouped = rebasedEdit.replacements.reduce((acc, item) => {
-		(acc[item.data.index] ||= []).push(item);
-		return acc;
-	}, [] as (AnnotatedStringReplacement<EditDataWithIndex>[] | undefined)[]);
-	const resultEdits: { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] = [];
-	for (let index = 0; index < grouped.length; index++) {
-		const group = grouped[index];
-		if (!group) {
-			continue;
-		}
-		const range = OffsetRange.fromTo(group[0].replaceRange.start, group[group.length - 1].replaceRange.endExclusive);
-		const newText = group.map((edit, i, a) => {
-			if (i > 0) {
-				return currentDocumentContent.substring(a[i - 1].replaceRange.endExclusive, edit.replaceRange.start) + edit.newText;
-			} else {
-				return edit.newText;
-			}
-		}).join('');
-		const resultEdit = StringReplacement.replace(range, newText);
-		if (!resultEdit.removeCommonSuffixAndPrefix(currentDocumentContent).isEmpty) {
-			resultEdits.push({ rebasedEdit: resultEdit, rebasedEditIndex: index });
-		}
-	}
-	if (resolution === 'strict' && resultEdits.length > 0 && new SingleEdits(originalEdits).apply(originalDocument) !== StringEdit.create(resultEdits.map(r => r.rebasedEdit)).apply(currentDocumentContent)) {
-		tracer.trace('Result consistency check failed.');
-		return 'inconsistentEdits';
-	}
-	return resultEdits;
-}
-
-export function checkEditConsistency(original: string, edit: StringEdit, current: string, tracer: ITracer, enabled = TROUBLESHOOT_EDIT_CONSISTENCY) {
-	if (!enabled) {
-		return true;
-	}
-	const consistent = edit.apply(original) === current;
-	if (!consistent) {
-		tracer.trace('Edit consistency check failed.');
-	}
-	return consistent;
-}
-
-export function tryRebaseStringEdits<T extends IEditData<T>>(content: string, ours: StringEdit, base: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs = {}): StringEdit | undefined {
-	return tryRebaseEdits(content, ours.mapData(r => new VoidEditData()), base, resolution, nesConfigs)?.toStringEdit();
-}
-
-function tryRebaseEdits<T extends IEditData<T>>(content: string, ours: AnnotatedStringEdit<T>, baseOrig: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs): AnnotatedStringEdit<T> | undefined {
-	const base = baseOrig.removeCommonSuffixAndPrefix(content);
-
-	const newEdits: AnnotatedStringReplacement<T>[] = [];
-
-	let baseIdx = 0;
-	let ourIdx = 0;
-	let offset = 0;
-
-	while (ourIdx < ours.replacements.length || baseIdx < base.replacements.length) {
-		// take the edit that starts first
-		const baseEdit = base.replacements[baseIdx];
-		const ourEdit = ours.replacements[ourIdx];
-
-		if (!ourEdit) {
-			if (resolution === 'strict') {
-				// baseEdit does not match but interleaves
-				return undefined;
-			}
-			// We processed all our edits
-			break;
-		} else if (!baseEdit) {
-			// no more edits from base
-			newEdits.push(ourEdit.delta(offset));
-			ourIdx++;
-		} else {
-			let ourE = ourEdit;
-			if (!ourE.replaceRange.containsRange(baseEdit.replaceRange)) {
-				// Try to shift our edit to include the base edit.
-				if (ourE.replaceRange.start > baseEdit.replaceRange.start) {
-					// Expand our edit to the left to include the base edit.
-					const added = content.substring(baseEdit.replaceRange.start, ourE.replaceRange.start);
-					const updated = added + ourE.newText;
-					// Remove the same text from the right.
-					if (updated.endsWith(added)) {
-						ourE = new AnnotatedStringReplacement(
-							OffsetRange.fromTo(baseEdit.replaceRange.start, ourE.replaceRange.endExclusive - added.length),
-							updated.substring(0, updated.length - added.length),
-							ourE.data,
-						);
-					}
-				}
-				// Skipping the case where there is another edit for now because we might have to merge with it first.
-				else if (ourIdx === ours.replacements.length - 1 && ourE.replaceRange.endExclusive < baseEdit.replaceRange.endExclusive) {
-					// Expand our edit to the right to include the base edit.
-					const added = content.substring(ourE.replaceRange.endExclusive, baseEdit.replaceRange.endExclusive);
-					const updated = ourE.newText + added;
-					// Remove the same text from the left.
-					if (updated.startsWith(added)) {
-						ourE = new AnnotatedStringReplacement(
-							OffsetRange.fromTo(ourE.replaceRange.start + added.length, baseEdit.replaceRange.endExclusive),
-							updated.substring(added.length),
-							ourE.data,
-						);
-					}
-				}
-			}
-			if (ourE.replaceRange.intersectsOrTouches(baseEdit.replaceRange)) {
-				if (ourE.replaceRange.containsRange(baseEdit.replaceRange) && ourE.newText.length >= baseEdit.newText.length) {
-					let delta = 0;
-					let ourNewTextOffset = 0;
-					let baseE = baseEdit;
-					let previousBaseE: StringReplacement | undefined;
-					while (baseE && ourE.replaceRange.containsRange(baseE.replaceRange)) {
-						ourNewTextOffset = agreementIndexOf(content, ourE, baseE, previousBaseE, ourNewTextOffset, resolution, nesConfigs);
-						if (ourNewTextOffset === -1) {
-							// Conflicting
-							return undefined;
-						}
-						delta += baseE.newText.length - baseE.replaceRange.length;
-						previousBaseE = baseE;
-						baseE = base.replacements[++baseIdx];
-					}
-					newEdits.push(new AnnotatedStringReplacement(
-						new OffsetRange(ourE.replaceRange.start + offset, ourE.replaceRange.endExclusive + offset + delta),
-						ourE.newText,
-						ourE.data,
-					));
-					ourIdx++;
-					offset += delta;
-				} else {
-					// Conflicting
-					return undefined;
-				}
-			} else if (ourEdit.replaceRange.start < baseEdit.replaceRange.start) {
-				// Our edit starts first
-				newEdits.push(new AnnotatedStringReplacement(
-					ourEdit.replaceRange.delta(offset),
-					ourEdit.newText,
-					ourEdit.data,
-				));
-				ourIdx++;
-			} else {
-				if (resolution === 'strict') {
-					// baseEdit does not match but interleaves
-					return undefined;
-				}
-				baseIdx++;
-				offset += baseEdit.newText.length - baseEdit.replaceRange.length;
-			}
-		}
-	}
-
-	return AnnotatedStringEdit.create(newEdits);
-}
-
-export const maxAgreementOffset = 10; // If the user's typing is more than this into the suggestion we consider it a miss.
-export const maxImperfectAgreementLength = 5; // If the user's typing is longer than this and the suggestion is not a perfect match we consider it a miss.
-
-function agreementIndexOf<T extends IEditData<T>>(content: string, ourE: AnnotatedStringReplacement<T>, baseE: StringReplacement, previousBaseE: StringReplacement | undefined, ourNewTextOffset: number, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs) {
-	const minStart = previousBaseE ? previousBaseE.replaceRange.endExclusive : ourE.replaceRange.start;
-	if (minStart < baseE.replaceRange.start) {
-		baseE = new StringReplacement(
-			OffsetRange.fromTo(minStart, baseE.replaceRange.endExclusive),
-			content.substring(minStart, baseE.replaceRange.start) + baseE.newText
-		);
-	}
-	const j = ourE.newText.indexOf(baseE.newText, ourNewTextOffset);
-	if (resolution === 'strict' && j > maxAgreementOffset) {
-		return -1;
-	}
-	if (resolution === 'strict' && j > 0 && baseE.newText.length > maxImperfectAgreementLength) {
-		return -1;
-	}
-	return j !== -1 ? j + baseE.newText.length : -1;
-}
-
-function computeDiff(original: string, modified: string, offset: number, editData: EditDataWithIndex, options: ILinesDiffComputerOptions): AnnotatedStringReplacement<EditDataWithIndex>[] | undefined {
-	const originalLines = original.split(/\r\n|\r|\n/);
-	const modifiedLines = modified.split(/\r\n|\r|\n/);
-	const diffComputer = new DefaultLinesDiffComputer();
-	const result = diffComputer.computeDiff(originalLines, modifiedLines, options);
-	if (result.hitTimeout) {
-		return undefined;
-	}
-
-	const originalText = new StringText(original);
-	const modifiedText = new StringText(modified);
-	return result.changes.map(change => (change.innerChanges || []).map(innerChange => {
-		const range = originalText.getTransformer().getOffsetRange(innerChange.originalRange);
-		const newText = modifiedText.getValueOfRange(innerChange.modifiedRange);
-		return new AnnotatedStringReplacement(range.delta(offset), newText, editData);
-	})).flat();
-}

package/src/_internal/extension/inlineEdits/common/ghNearbyNesProvider.tsx

@@ -1,163 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { StatelessNextEditDocument } from '../../../platform/inlineEdits/common/statelessNextEditProvider';
-import { LineEdit } from '../../../util/vs/editor/common/core/edits/lineEdit';
-import { StringEdit } from '../../../util/vs/editor/common/core/edits/stringEdit';
-import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
-
-const N_GRAM_UNDO_RATIO_TO_FILTER_OUT = 0.7;
-
-export function editWouldDeleteWhatWasJustInserted(activeDocument: StatelessNextEditDocument, lineEdit: LineEdit) {
-	let edit = lineEdit.toEdit(activeDocument.documentAfterEdits);
-	// ! important: reduce it to the minimal set of changes
-	edit = edit.normalizeOnSource(activeDocument.documentAfterEdits.value);
-	if (!editIsDeletion(edit)) {
-		return false;
-	}
-	// We are deleting something. Is it what was just inserted?
-	for (let i = activeDocument.recentEdits.edits.length - 1; i >= 0; i--) {
-		const recentEdit = activeDocument.recentEdits.edits[i];
-		const rebaseResult = edit.tryRebase(recentEdit);
-		if (!rebaseResult) {
-			// the edit we want to do cannot be rebased, which indicates that it would interfere with a recent edit
-			return true;
-		}
-		edit = rebaseResult;
-	}
-	return false;
-}
-
-function editIsDeletion(edit: StringEdit): boolean {
-	const deletedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.replaceRange.length, 0);
-	const insertedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.newText.length, 0);
-	return insertedChars === 0 && deletedChars > 0;
-}
-
-/**
- * Represents information loss/gain (4-grams) via an edit.
- */
-export class InformationDelta {
-
-	constructor(
-		public readonly inserted: Set<string> = new Set<string>(),
-		public readonly deleted: Set<string> = new Set<string>()
-	) { }
-
-	combine(other: InformationDelta) {
-		return new InformationDelta(setUnion(this.inserted, other.inserted), setUnion(this.deleted, other.deleted));
-	}
-
-	isUndoneBy(other: InformationDelta) {
-		const otherReallyNewInsertions = setMinus(other.inserted, other.deleted);
-		const otherReallyDeleted = setMinus(other.deleted, other.inserted);
-
-		const otherReallyDeletesMyInserts = setIntersectionCount(otherReallyDeleted, this.inserted);
-		const otherReallyInsertsMyDeletes = setIntersectionCount(otherReallyNewInsertions, this.deleted);
-
-		if (otherReallyDeleted.size > 6 && otherReallyDeletesMyInserts / otherReallyDeleted.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
-			return true;
-		}
-
-		if (otherReallyNewInsertions.size > 6 && otherReallyInsertsMyDeletes / otherReallyNewInsertions.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
-			return true;
-		}
-
-		return false;
-	}
-}
-
-export function getInformationDelta(source: string, edit: StringEdit): InformationDelta {
-	const inserted = new Set<string>();
-	const deleted = new Set<string>();
-	const tryAddDeleted = (deletedRange: OffsetRange | undefined) => {
-		if (!deletedRange) {
-			return;
-		}
-		const deletedText = source.substring(deletedRange.start, deletedRange.endExclusive);
-		for (let line of deletedText.split(/\r\n|\r|\n/)) {
-			line = line.trim();
-			for (const piece of to4grams(line)) {
-				deleted.add(piece);
-			}
-		}
-	};
-	const tryAddInserted = (insertedText: string) => {
-		for (let line of insertedText.split(/\r\n|\r|\n/)) {
-			line = line.trim();
-			for (const piece of to4grams(line)) {
-				inserted.add(piece);
-			}
-		}
-	};
-	for (const e of edit.replacements) {
-		const e1 = e.removeCommonPrefix(source).removeCommonSuffix(source);
-		const e2 = e.removeCommonSuffix(source).removeCommonPrefix(source);
-		if (e1.isEmpty) {
-			continue;
-		}
-		tryAddDeleted(e1.replaceRange);
-		tryAddDeleted(e2.replaceRange);
-		tryAddDeleted(e1.replaceRange.intersect(e2.replaceRange));
-
-		// tryAddInserted(e1.newText);
-		// tryAddInserted(e2.newText);
-		// e1 might have a suffix overlap with the prefix of e1
-		tryAddInserted(trimOverlap(e1.newText, e2.newText));
-	}
-	return new InformationDelta(inserted, deleted);
-}
-
-function trimOverlap(stringToEliminateEnd: string, stringToEliminateStart: string): string {
-	const length = Math.min(stringToEliminateEnd.length, stringToEliminateStart.length);
-	for (let trimLength = 0; trimLength < length; trimLength++) {
-		const str1 = stringToEliminateEnd.slice(0, stringToEliminateEnd.length - trimLength);
-		const str2 = stringToEliminateStart.slice(trimLength);
-		if (str1 === str2) {
-			return str1;
-		}
-	}
-	return '';
-}
-
-function to4grams(text: string) {
-	const result: string[] = [];
-	for (let i = 4; i < text.length; i++) {
-		const ngram = text.slice(i - 4, i);
-		result.push(ngram);
-	}
-	return result;
-}
-
-function setUnion(a: Set<string>, b: Set<string>): Set<string> {
-	const result = new Set<string>();
-	for (const el of a) {
-		result.add(el);
-	}
-	for (const el of b) {
-		result.add(el);
-	}
-	return result;
-}
-
-function setMinus(a: Set<string>, b: Set<string>): Set<string> {
-	const result = new Set<string>();
-	for (const el of a) {
-		if (!b.has(el)) {
-			result.add(el);
-		}
-	}
-	return result;
-}
-
-function setIntersectionCount(a: Set<string>, b: Set<string>): number {
-	let result = 0;
-	for (const el of a) {
-		if (b.has(el)) {
-			result++;
-		}
-	}
-	return result;
-}

package/src/_internal/extension/inlineEdits/common/nearbyCursorInlineEditProvider.ts

@@ -1,225 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { StatelessNextEditDocument } from '../../../platform/inlineEdits/common/statelessNextEditProvider';
-import { ChoiceLogProbs } from '../../../platform/networking/common/openai';
-import { BugIndicatingError } from '../../../util/vs/base/common/errors';
-import { Range } from '../../../util/vs/editor/common/core/range';
-import { OffsetRange, OffsetRangeSet } from '../../../util/vs/editor/common/core/ranges/offsetRange';
-
-/**
- * Read the selection from the document, otherwise deduce it from the last edit.
- */
-export function getOrDeduceSelectionFromLastEdit(activeDoc: StatelessNextEditDocument): Range | null {
-	const origin = new OffsetRange(0, 0);
-	if (activeDoc.lastSelectionInAfterEdit && !activeDoc.lastSelectionInAfterEdit.equals(origin)) {
-		return activeDoc.documentAfterEdits.getTransformer().getRange(activeDoc.lastSelectionInAfterEdit);
-	}
-
-	const selectionRange = deduceSelectionFromLastEdit(activeDoc);
-	return selectionRange;
-}
-
-function deduceSelectionFromLastEdit(activeDoc: StatelessNextEditDocument): Range | null {
-	const mostRecentEdit = activeDoc.recentEdits.edits.at(-1);
-	if (mostRecentEdit === undefined) {
-		return null;
-	}
-
-	const mostRecentSingleEdit = mostRecentEdit.replacements.at(-1);
-	if (mostRecentSingleEdit === undefined) {
-		return null;
-	}
-
-	const offsetRange = mostRecentSingleEdit.replaceRange;
-	const newText = mostRecentSingleEdit.newText;
-	const change = newText.length - offsetRange.length;
-	const newOffset = offsetRange.endExclusive + change;
-
-	const selectionRange = activeDoc.documentAfterEdits.getTransformer().getRange(new OffsetRange(newOffset, newOffset));
-
-	return selectionRange;
-}
-
-type Tokens = Token<number>[];
-
-export class Token<T> {
-	public readonly range: OffsetRange;
-
-	get id(): string {
-		return this.text + '_' + this.range.toString();
-	}
-
-	constructor(public readonly text: string, public readonly value: T, offset: number) {
-		this.range = new OffsetRange(offset, offset + text.length);
-	}
-
-	public equals(other: Token<T>): boolean {
-		return this.range.equals(other.range) && this.text === other.text;
-	}
-
-	public deltaOffset(offset: number): Token<T> {
-		return new Token(this.text, this.value, this.range.start + offset);
-	}
-}
-
-export function clipTokensToRange(tokens: Tokens, range: OffsetRange): Tokens {
-	return tokens.filter(token => range.intersects(token.range));
-}
-
-export function clipTokensToRangeAndAdjustOffsets(tokens: Tokens, range: OffsetRange): Tokens {
-	return clipTokensToRange(tokens, range).map(token => token.deltaOffset(-range.start));
-}
-
-export function removeTokensInRangeAndAdjustOffsets(tokens: Tokens, range: OffsetRange): Tokens {
-	const adjustedTokens: Tokens = [];
-	for (let token of tokens) {
-		// remove tokens inside the range
-		if (range.containsRange(token.range)) {
-			continue;
-		}
-		// adjust the token offset
-		if (token.range.start > range.start) {
-			token = token.deltaOffset(-range.length);
-		}
-
-		adjustedTokens.push(token);
-	}
-
-	return adjustedTokens;
-}
-
-export function getTokensFromLogProbs(logProbs: ChoiceLogProbs, offset: number): Tokens {
-	let acc = offset;
-	return logProbs.content.map(tokenContent => {
-		const token = new Token(tokenContent.token, tokenContent.logprob, acc);
-		acc += token.range.length;
-		return token;
-	});
-}
-
-export class LineWithTokens {
-
-	static stringEquals(a: LineWithTokens, b: LineWithTokens): boolean {
-		return a._text === b._text;
-	}
-
-	static fromText(text: string, tokens: Tokens | undefined): LineWithTokens[] {
-		tokens = tokens ?? [];
-
-		const lines: LineWithTokens[] = [];
-		while (true) {
-			const eolIdxWith = text.indexOf('\r\n');
-			const eolIdxWithout = text.indexOf('\n');
-			const eolIdx = (eolIdxWith === -1 ? eolIdxWithout : (eolIdxWithout === -1 ? eolIdxWith : Math.min(eolIdxWith, eolIdxWithout)));
-			const eol = (eolIdxWith !== -1 ? '\r\n' : (eolIdxWithout === -1 ? undefined : '\n'));
-
-			if (eol === undefined) {
-				lines.push(new LineWithTokens(text, tokens, '\n'));
-				break;
-			}
-
-			const lineLength = eolIdx + eol.length;
-			const line = text.substring(0, eolIdx);
-			const lineTokensWithBoundary = tokens.filter(t => t.range.start < lineLength && t.range.endExclusive > 0);
-			lines.push(new LineWithTokens(line, lineTokensWithBoundary, eol));
-
-			text = text.substring(lineLength);
-			tokens = tokens.map(t => t.deltaOffset(-lineLength)).filter(t => t.range.endExclusive > 0);
-		}
-
-		return lines;
-	}
-
-	get text(): string { return this._text; }
-	get tokens(): Tokens { return this._tokens; }
-	get length(): number { return this._text.length; }
-	get lengthWithEOL(): number { return this._text.length + this._eol.length; }
-	get eol(): '\n' | '\r\n' { return this._eol; }
-
-	constructor(
-		private readonly _text: string,
-		private readonly _tokens: Tokens,
-		private readonly _eol: '\n' | '\r\n'
-	) { }
-
-	trim() {
-		return this.trimStart().trimEnd();
-	}
-
-	trimStart() {
-		const lineStartTrimmed = this._text.trimStart();
-		const trimmedLength = this._text.length - lineStartTrimmed.length;
-		const tokensUpdated = this._tokens.map(t => t.deltaOffset(-trimmedLength)).filter(t => t.range.endExclusive > 0);
-		return new LineWithTokens(lineStartTrimmed, tokensUpdated, this._eol);
-	}
-
-	trimEnd() {
-		const lineEndTrimmed = this._text.trimEnd();
-		const tokensUpdated = this._tokens.filter(t => t.range.start < lineEndTrimmed.length);
-		return new LineWithTokens(lineEndTrimmed, tokensUpdated, this._eol);
-	}
-
-	substring(start: number, end: number): LineWithTokens {
-		const lineSubstring = this._text.substring(start, end);
-		const tokensUpdated = this._tokens.map(t => t.deltaOffset(-start)).filter(t => t.range.endExclusive > 0 && t.range.start < lineSubstring.length);
-		return new LineWithTokens(lineSubstring, tokensUpdated, this._eol);
-	}
-
-	stringEquals(other: LineWithTokens): boolean {
-		return LineWithTokens.stringEquals(this, other);
-	}
-
-	equals(other: LineWithTokens): boolean {
-		return this._text === other.text
-			&& this._tokens.length === other.tokens.length
-			&& this._tokens.every((t, i) => t.equals(other.tokens[i]));
-	}
-
-	dropTokens(tokens: Tokens): LineWithTokens {
-		return new LineWithTokens(this._text, this._tokens.filter(t => !tokens.some(token => t.equals(token))), this._eol);
-	}
-
-	findTokens(fn: (token: Token<number>) => boolean): Token<number>[] {
-		return this._tokens.filter(fn);
-	}
-}
-
-export function getTokensFromLinesWithTokens(lines: LineWithTokens[]): Tokens {
-	let offset = 0;
-
-	const tokens: Tokens = [];
-	for (const line of lines) {
-		const textLine = line.text + line.eol;
-		tokens.push(...line.tokens.map(t => t.deltaOffset(offset)));
-		offset += textLine.length;
-	}
-
-	const tokensDeduplicated: Tokens = [];
-	const tokensSeen = new Set<string>();
-	for (const token of tokens) {
-		if (!tokensSeen.has(token.id)) {
-			tokensSeen.add(token.id);
-			tokensDeduplicated.push(token);
-		}
-	}
-
-	return tokensDeduplicated;
-}
-
-export function mergeOffsetRangesAtDistance(ranges: OffsetRange[], distance: number): OffsetRange[] {
-	if (distance < 0) {
-		throw new BugIndicatingError('Distance must be positive');
-	}
-
-	const rangesGrown = ranges.map(r => new OffsetRange(r.start - distance, r.endExclusive + distance));
-
-	const set = new OffsetRangeSet();
-	for (const range of rangesGrown) {
-		set.addRange(range);
-	}
-
-	return set.ranges.map(r => new OffsetRange(r.start + distance, r.endExclusive - distance));
-}