@vscode/chat-lib 0.0.1 → 0.0.2

package/src/_internal/extension/common/constants.ts

@@ -1,90 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { ChatLocation } from '../../platform/chat/common/commonTypes';
-
-export const enum Intent {
-	Explain = 'explain',
-	Review = 'review',
-	Tests = 'tests',
-	Fix = 'fix',
-	New = 'new',
-	NewNotebook = 'newNotebook',
-	notebookEditor = 'notebookEditor',
-	Search = 'search',
-	SemanticSearch = 'semanticSearch',
-	Terminal = 'terminal',
-	TerminalExplain = 'terminalExplain',
-	VSCode = 'vscode',
-	Workspace = 'workspace',
-	Unknown = 'unknown',
-	StartDebugging = 'startDebugging',
-	SetupTests = 'setupTests',
-	Editor = 'editor',
-	Doc = 'doc',
-	Edit = 'edit',
-	Edit2 = 'edit2',
-	Agent = 'editAgent',
-	Generate = 'generate',
-	SearchPanel = 'searchPanel',
-	SearchKeywords = 'searchKeywords',
-	AskAgent = 'askAgent',
-	ChatReplay = 'chatReplay'
-}
-
-export const GITHUB_PLATFORM_AGENT = 'github.copilot-dynamic.platform';
-
-// TODO@jrieken THIS IS WEIRD. We should read this from package.json
-export const agentsToCommands: Partial<Record<Intent, Record<string, Intent>>> = {
-	[Intent.Workspace]: {
-		'explain': Intent.Explain,
-		'edit': Intent.Edit,
-		'review': Intent.Review,
-		'tests': Intent.Tests,
-		'fix': Intent.Fix,
-		'new': Intent.New,
-		'newNotebook': Intent.NewNotebook,
-		'semanticSearch': Intent.SemanticSearch,
-		'setupTests': Intent.SetupTests,
-	},
-	[Intent.VSCode]: {
-		'search': Intent.Search,
-		'startDebugging': Intent.StartDebugging,
-	},
-	[Intent.Terminal]: {
-		'explain': Intent.TerminalExplain
-	},
-	[Intent.Editor]: {
-		'doc': Intent.Doc,
-		'fix': Intent.Fix,
-		'explain': Intent.Explain,
-		'review': Intent.Review,
-		'tests': Intent.Tests,
-		'edit': Intent.Edit,
-		'generate': Intent.Generate
-	},
-	[Intent.ChatReplay]: {
-		'chatReplay': Intent.ChatReplay
-	}
-};
-
-// TODO@roblourens gotta tighten up the terminology of "commands", "intents", etc...
-export function getAgentForIntent(intentId: Intent, location: ChatLocation): { agent: string; command?: string } | undefined {
-	if (Object.keys(agentsToCommands).includes(intentId)) {
-		return { agent: intentId };
-	}
-
-	for (const [agent, commands] of Object.entries(agentsToCommands)) {
-		if (location === ChatLocation.Editor && agent !== Intent.Editor) {
-			continue;
-		}
-
-		if (Object.values(commands).includes(intentId)) {
-			return { agent, command: intentId };
-		}
-	}
-}
-
-export const EXTENSION_ID = 'GitHub.copilot-chat';

package/src/_internal/extension/inlineEdits/common/delayer.ts

@@ -1,108 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { ConfigKey, IConfigurationService } from '../../../platform/configuration/common/configurationService';
-import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
-
-export class DelaySession {
-	private extraDebounce = 0;
-
-	constructor(
-		private baseDebounceTime: number,
-		private readonly expectedTotalTime: number | undefined,
-		private readonly providerInvocationTime: number = Date.now(),
-	) {
-	}
-
-	public setExtraDebounce(extraDebounce: number): void {
-		this.extraDebounce = extraDebounce;
-	}
-
-	getDebounceTime() {
-		const expectedDebounceTime = this.expectedTotalTime === undefined
-			? this.baseDebounceTime + this.extraDebounce
-			: Math.min(this.baseDebounceTime + this.extraDebounce, this.expectedTotalTime);
-
-		const timeAlreadySpent = Date.now() - this.providerInvocationTime;
-		const actualDebounceTime = Math.max(0, expectedDebounceTime - timeAlreadySpent);
-
-		return actualDebounceTime;
-	}
-
-	getArtificialDelay() {
-		if (this.expectedTotalTime === undefined) {
-			return 0;
-		}
-
-		const timeAlreadySpent = Date.now() - this.providerInvocationTime;
-		const delay = Math.max(0, this.expectedTotalTime - timeAlreadySpent);
-		return delay;
-	}
-}
-
-export class Delayer {
-
-	private _recentUserActions: { time: number; kind: 'accepted' | 'rejected' }[] = [];
-
-	constructor(
-		@IConfigurationService private readonly _configurationService: IConfigurationService,
-		@IExperimentationService private readonly _experimentationService: IExperimentationService,
-	) {
-	}
-
-	public createDelaySession(requestTime: number | undefined): DelaySession {
-		const baseDebounceTime = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.InlineEditsDebounce, this._experimentationService);
-
-		const backoffDebounceEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.InlineEditsBackoffDebounceEnabled, this._experimentationService);
-		const expectedTotalTime = backoffDebounceEnabled ? this._getExpectedTotalTime(baseDebounceTime) : undefined;
-
-		return new DelaySession(baseDebounceTime, expectedTotalTime, requestTime);
-	}
-
-	public handleAcceptance(): void {
-		this._recordUserAction('accepted');
-	}
-
-	public handleRejection(): void {
-		this._recordUserAction('rejected');
-	}
-
-	private _recordUserAction(kind: 'accepted' | 'rejected') {
-		this._recentUserActions.push({ time: Date.now(), kind });
-		// keep at most 10 user actions
-		this._recentUserActions = this._recentUserActions.slice(-10);
-	}
-
-	private _getExpectedTotalTime(baseDebounceTime: number): number {
-		const DEBOUNCE_DECAY_TIME_MS = 10 * 60 * 1000; // 10 minutes
-		const MAX_DEBOUNCE_TIME = 3000; // 3 seconds
-		const MIN_DEBOUNCE_TIME = 50; // 50 ms
-		const REJECTION_WEIGHT = 1.5;
-		const ACCEPTANCE_WEIGHT = 0.8;
-		const now = Date.now();
-		let multiplier = 1;
-
-		// Calculate impact of each action with time decay
-		for (const action of this._recentUserActions) {
-			const timeSinceAction = now - action.time;
-			if (timeSinceAction > DEBOUNCE_DECAY_TIME_MS) {
-				continue;
-			}
-
-			// Exponential decay: impact decreases as time passes
-			const decayFactor = Math.exp(-timeSinceAction / DEBOUNCE_DECAY_TIME_MS);
-			const actionWeight = action.kind === 'rejected' ? REJECTION_WEIGHT : ACCEPTANCE_WEIGHT;
-			multiplier *= 1 + ((actionWeight - 1) * decayFactor);
-		}
-
-		let debounceTime = baseDebounceTime * multiplier;
-
-		// Clamp the debounce time to reasonable bounds
-		debounceTime = Math.min(MAX_DEBOUNCE_TIME, Math.max(MIN_DEBOUNCE_TIME, debounceTime));
-
-		return debounceTime;
-	}
-
-}

package/src/_internal/extension/inlineEdits/common/editRebase.ts

@@ -1,268 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { SingleEdits } from '../../../platform/inlineEdits/common/dataTypes/edit';
-import * as errors from '../../../util/common/errors';
-import { ITracer } from '../../../util/common/tracing';
-import { AnnotatedStringEdit, AnnotatedStringReplacement, IEditData, StringEdit, StringReplacement, VoidEditData } from '../../../util/vs/editor/common/core/edits/stringEdit';
-import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
-import { StringText } from '../../../util/vs/editor/common/core/text/abstractText';
-import { DefaultLinesDiffComputer } from '../../../util/vs/editor/common/diff/defaultLinesDiffComputer/defaultLinesDiffComputer';
-import { ILinesDiffComputerOptions } from '../../../util/vs/editor/common/diff/linesDiffComputer';
-
-const TROUBLESHOOT_EDIT_CONSISTENCY = false;
-
-export interface NesRebaseConfigs {
-}
-
-export class EditDataWithIndex implements IEditData<EditDataWithIndex> {
-	constructor(
-		public readonly index: number
-	) { }
-
-	join(data: EditDataWithIndex): EditDataWithIndex | undefined {
-		if (this.index !== data.index) {
-			return undefined;
-		}
-		return this;
-	}
-}
-
-export function tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSince: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs = {}): { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] | 'outsideEditWindow' | 'rebaseFailed' | 'error' | 'inconsistentEdits' {
-	const start = Date.now();
-	try {
-		return _tryRebase(originalDocument, editWindow, originalEdits, detailedEdits, userEditSince, currentDocumentContent, currentSelection, resolution, tracer, nesConfigs);
-	} catch (err) {
-		tracer.trace(`Rebase error: ${errors.toString(err)}`);
-		return 'error';
-	} finally {
-		tracer.trace(`Rebase duration: ${Date.now() - start}ms`);
-	}
-}
-
-function _tryRebase(originalDocument: string, editWindow: OffsetRange | undefined, originalEdits: readonly StringReplacement[], detailedEdits: AnnotatedStringReplacement<EditDataWithIndex>[][], userEditSinceOrig: StringEdit, currentDocumentContent: string, currentSelection: readonly OffsetRange[], resolution: 'strict' | 'lenient', tracer: ITracer, nesConfigs: NesRebaseConfigs) {
-	if (!checkEditConsistency(originalDocument, userEditSinceOrig, currentDocumentContent, tracer, true)) {
-		return 'inconsistentEdits';
-	}
-	const userEditSince = userEditSinceOrig.removeCommonSuffixAndPrefix(originalDocument);
-	const cursorRange = currentSelection[0];
-	if (editWindow && cursorRange) {
-		const updatedEditWindow = userEditSince.applyToOffsetRangeOrUndefined(editWindow);
-		if (!updatedEditWindow?.containsRange(cursorRange)) {
-			return 'outsideEditWindow';
-		}
-	}
-	if (detailedEdits.length < originalEdits.length) {
-		let intermediateDocument = originalDocument;
-		for (let index = 0; index < detailedEdits.length; index++) {
-			const edit = originalEdits[index];
-			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
-		}
-		for (let index = detailedEdits.length; index < originalEdits.length; index++) {
-			const edit = originalEdits[index];
-			const editData = new EditDataWithIndex(index);
-			detailedEdits[index] = computeDiff(edit.replaceRange.substring(intermediateDocument), edit.newText, edit.replaceRange.start, editData, {
-				ignoreTrimWhitespace: false,
-				computeMoves: false,
-				extendToSubwords: true,
-				maxComputationTimeMs: 500,
-			}) || [new AnnotatedStringReplacement(edit.replaceRange, edit.newText, editData)];
-			intermediateDocument = StringEdit.single(edit).apply(intermediateDocument);
-		}
-	}
-	const diffedEdit = AnnotatedStringEdit.compose(detailedEdits.map(edits => AnnotatedStringEdit.create(edits)));
-	const rebasedEdit = tryRebaseEdits(originalDocument, diffedEdit, userEditSince, resolution, nesConfigs);
-	if (!rebasedEdit) {
-		return 'rebaseFailed';
-	}
-	const grouped = rebasedEdit.replacements.reduce((acc, item) => {
-		(acc[item.data.index] ||= []).push(item);
-		return acc;
-	}, [] as (AnnotatedStringReplacement<EditDataWithIndex>[] | undefined)[]);
-	const resultEdits: { rebasedEdit: StringReplacement; rebasedEditIndex: number }[] = [];
-	for (let index = 0; index < grouped.length; index++) {
-		const group = grouped[index];
-		if (!group) {
-			continue;
-		}
-		const range = OffsetRange.fromTo(group[0].replaceRange.start, group[group.length - 1].replaceRange.endExclusive);
-		const newText = group.map((edit, i, a) => {
-			if (i > 0) {
-				return currentDocumentContent.substring(a[i - 1].replaceRange.endExclusive, edit.replaceRange.start) + edit.newText;
-			} else {
-				return edit.newText;
-			}
-		}).join('');
-		const resultEdit = StringReplacement.replace(range, newText);
-		if (!resultEdit.removeCommonSuffixAndPrefix(currentDocumentContent).isEmpty) {
-			resultEdits.push({ rebasedEdit: resultEdit, rebasedEditIndex: index });
-		}
-	}
-	if (resolution === 'strict' && resultEdits.length > 0 && new SingleEdits(originalEdits).apply(originalDocument) !== StringEdit.create(resultEdits.map(r => r.rebasedEdit)).apply(currentDocumentContent)) {
-		tracer.trace('Result consistency check failed.');
-		return 'inconsistentEdits';
-	}
-	return resultEdits;
-}
-
-export function checkEditConsistency(original: string, edit: StringEdit, current: string, tracer: ITracer, enabled = TROUBLESHOOT_EDIT_CONSISTENCY) {
-	if (!enabled) {
-		return true;
-	}
-	const consistent = edit.apply(original) === current;
-	if (!consistent) {
-		tracer.trace('Edit consistency check failed.');
-	}
-	return consistent;
-}
-
-export function tryRebaseStringEdits<T extends IEditData<T>>(content: string, ours: StringEdit, base: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs = {}): StringEdit | undefined {
-	return tryRebaseEdits(content, ours.mapData(r => new VoidEditData()), base, resolution, nesConfigs)?.toStringEdit();
-}
-
-function tryRebaseEdits<T extends IEditData<T>>(content: string, ours: AnnotatedStringEdit<T>, baseOrig: StringEdit, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs): AnnotatedStringEdit<T> | undefined {
-	const base = baseOrig.removeCommonSuffixAndPrefix(content);
-
-	const newEdits: AnnotatedStringReplacement<T>[] = [];
-
-	let baseIdx = 0;
-	let ourIdx = 0;
-	let offset = 0;
-
-	while (ourIdx < ours.replacements.length || baseIdx < base.replacements.length) {
-		// take the edit that starts first
-		const baseEdit = base.replacements[baseIdx];
-		const ourEdit = ours.replacements[ourIdx];
-
-		if (!ourEdit) {
-			if (resolution === 'strict') {
-				// baseEdit does not match but interleaves
-				return undefined;
-			}
-			// We processed all our edits
-			break;
-		} else if (!baseEdit) {
-			// no more edits from base
-			newEdits.push(ourEdit.delta(offset));
-			ourIdx++;
-		} else {
-			let ourE = ourEdit;
-			if (!ourE.replaceRange.containsRange(baseEdit.replaceRange)) {
-				// Try to shift our edit to include the base edit.
-				if (ourE.replaceRange.start > baseEdit.replaceRange.start) {
-					// Expand our edit to the left to include the base edit.
-					const added = content.substring(baseEdit.replaceRange.start, ourE.replaceRange.start);
-					const updated = added + ourE.newText;
-					// Remove the same text from the right.
-					if (updated.endsWith(added)) {
-						ourE = new AnnotatedStringReplacement(
-							OffsetRange.fromTo(baseEdit.replaceRange.start, ourE.replaceRange.endExclusive - added.length),
-							updated.substring(0, updated.length - added.length),
-							ourE.data,
-						);
-					}
-				}
-				// Skipping the case where there is another edit for now because we might have to merge with it first.
-				else if (ourIdx === ours.replacements.length - 1 && ourE.replaceRange.endExclusive < baseEdit.replaceRange.endExclusive) {
-					// Expand our edit to the right to include the base edit.
-					const added = content.substring(ourE.replaceRange.endExclusive, baseEdit.replaceRange.endExclusive);
-					const updated = ourE.newText + added;
-					// Remove the same text from the left.
-					if (updated.startsWith(added)) {
-						ourE = new AnnotatedStringReplacement(
-							OffsetRange.fromTo(ourE.replaceRange.start + added.length, baseEdit.replaceRange.endExclusive),
-							updated.substring(added.length),
-							ourE.data,
-						);
-					}
-				}
-			}
-			if (ourE.replaceRange.intersectsOrTouches(baseEdit.replaceRange)) {
-				if (ourE.replaceRange.containsRange(baseEdit.replaceRange) && ourE.newText.length >= baseEdit.newText.length) {
-					let delta = 0;
-					let ourNewTextOffset = 0;
-					let baseE = baseEdit;
-					let previousBaseE: StringReplacement | undefined;
-					while (baseE && ourE.replaceRange.containsRange(baseE.replaceRange)) {
-						ourNewTextOffset = agreementIndexOf(content, ourE, baseE, previousBaseE, ourNewTextOffset, resolution, nesConfigs);
-						if (ourNewTextOffset === -1) {
-							// Conflicting
-							return undefined;
-						}
-						delta += baseE.newText.length - baseE.replaceRange.length;
-						previousBaseE = baseE;
-						baseE = base.replacements[++baseIdx];
-					}
-					newEdits.push(new AnnotatedStringReplacement(
-						new OffsetRange(ourE.replaceRange.start + offset, ourE.replaceRange.endExclusive + offset + delta),
-						ourE.newText,
-						ourE.data,
-					));
-					ourIdx++;
-					offset += delta;
-				} else {
-					// Conflicting
-					return undefined;
-				}
-			} else if (ourEdit.replaceRange.start < baseEdit.replaceRange.start) {
-				// Our edit starts first
-				newEdits.push(new AnnotatedStringReplacement(
-					ourEdit.replaceRange.delta(offset),
-					ourEdit.newText,
-					ourEdit.data,
-				));
-				ourIdx++;
-			} else {
-				if (resolution === 'strict') {
-					// baseEdit does not match but interleaves
-					return undefined;
-				}
-				baseIdx++;
-				offset += baseEdit.newText.length - baseEdit.replaceRange.length;
-			}
-		}
-	}
-
-	return AnnotatedStringEdit.create(newEdits);
-}
-
-export const maxAgreementOffset = 10; // If the user's typing is more than this into the suggestion we consider it a miss.
-export const maxImperfectAgreementLength = 5; // If the user's typing is longer than this and the suggestion is not a perfect match we consider it a miss.
-
-function agreementIndexOf<T extends IEditData<T>>(content: string, ourE: AnnotatedStringReplacement<T>, baseE: StringReplacement, previousBaseE: StringReplacement | undefined, ourNewTextOffset: number, resolution: 'strict' | 'lenient', nesConfigs: NesRebaseConfigs) {
-	const minStart = previousBaseE ? previousBaseE.replaceRange.endExclusive : ourE.replaceRange.start;
-	if (minStart < baseE.replaceRange.start) {
-		baseE = new StringReplacement(
-			OffsetRange.fromTo(minStart, baseE.replaceRange.endExclusive),
-			content.substring(minStart, baseE.replaceRange.start) + baseE.newText
-		);
-	}
-	const j = ourE.newText.indexOf(baseE.newText, ourNewTextOffset);
-	if (resolution === 'strict' && j > maxAgreementOffset) {
-		return -1;
-	}
-	if (resolution === 'strict' && j > 0 && baseE.newText.length > maxImperfectAgreementLength) {
-		return -1;
-	}
-	return j !== -1 ? j + baseE.newText.length : -1;
-}
-
-function computeDiff(original: string, modified: string, offset: number, editData: EditDataWithIndex, options: ILinesDiffComputerOptions): AnnotatedStringReplacement<EditDataWithIndex>[] | undefined {
-	const originalLines = original.split(/\r\n|\r|\n/);
-	const modifiedLines = modified.split(/\r\n|\r|\n/);
-	const diffComputer = new DefaultLinesDiffComputer();
-	const result = diffComputer.computeDiff(originalLines, modifiedLines, options);
-	if (result.hitTimeout) {
-		return undefined;
-	}
-
-	const originalText = new StringText(original);
-	const modifiedText = new StringText(modified);
-	return result.changes.map(change => (change.innerChanges || []).map(innerChange => {
-		const range = originalText.getTransformer().getOffsetRange(innerChange.originalRange);
-		const newText = modifiedText.getValueOfRange(innerChange.modifiedRange);
-		return new AnnotatedStringReplacement(range.delta(offset), newText, editData);
-	})).flat();
-}

package/src/_internal/extension/inlineEdits/common/ghNearbyNesProvider.tsx

@@ -1,163 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { StatelessNextEditDocument } from '../../../platform/inlineEdits/common/statelessNextEditProvider';
-import { LineEdit } from '../../../util/vs/editor/common/core/edits/lineEdit';
-import { StringEdit } from '../../../util/vs/editor/common/core/edits/stringEdit';
-import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
-
-const N_GRAM_UNDO_RATIO_TO_FILTER_OUT = 0.7;
-
-export function editWouldDeleteWhatWasJustInserted(activeDocument: StatelessNextEditDocument, lineEdit: LineEdit) {
-	let edit = lineEdit.toEdit(activeDocument.documentAfterEdits);
-	// ! important: reduce it to the minimal set of changes
-	edit = edit.normalizeOnSource(activeDocument.documentAfterEdits.value);
-	if (!editIsDeletion(edit)) {
-		return false;
-	}
-	// We are deleting something. Is it what was just inserted?
-	for (let i = activeDocument.recentEdits.edits.length - 1; i >= 0; i--) {
-		const recentEdit = activeDocument.recentEdits.edits[i];
-		const rebaseResult = edit.tryRebase(recentEdit);
-		if (!rebaseResult) {
-			// the edit we want to do cannot be rebased, which indicates that it would interfere with a recent edit
-			return true;
-		}
-		edit = rebaseResult;
-	}
-	return false;
-}
-
-function editIsDeletion(edit: StringEdit): boolean {
-	const deletedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.replaceRange.length, 0);
-	const insertedChars = edit.replacements.reduce((acc, singleEdit) => acc + singleEdit.newText.length, 0);
-	return insertedChars === 0 && deletedChars > 0;
-}
-
-/**
- * Represents information loss/gain (4-grams) via an edit.
- */
-export class InformationDelta {
-
-	constructor(
-		public readonly inserted: Set<string> = new Set<string>(),
-		public readonly deleted: Set<string> = new Set<string>()
-	) { }
-
-	combine(other: InformationDelta) {
-		return new InformationDelta(setUnion(this.inserted, other.inserted), setUnion(this.deleted, other.deleted));
-	}
-
-	isUndoneBy(other: InformationDelta) {
-		const otherReallyNewInsertions = setMinus(other.inserted, other.deleted);
-		const otherReallyDeleted = setMinus(other.deleted, other.inserted);
-
-		const otherReallyDeletesMyInserts = setIntersectionCount(otherReallyDeleted, this.inserted);
-		const otherReallyInsertsMyDeletes = setIntersectionCount(otherReallyNewInsertions, this.deleted);
-
-		if (otherReallyDeleted.size > 6 && otherReallyDeletesMyInserts / otherReallyDeleted.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
-			return true;
-		}
-
-		if (otherReallyNewInsertions.size > 6 && otherReallyInsertsMyDeletes / otherReallyNewInsertions.size > N_GRAM_UNDO_RATIO_TO_FILTER_OUT) {
-			return true;
-		}
-
-		return false;
-	}
-}
-
-export function getInformationDelta(source: string, edit: StringEdit): InformationDelta {
-	const inserted = new Set<string>();
-	const deleted = new Set<string>();
-	const tryAddDeleted = (deletedRange: OffsetRange | undefined) => {
-		if (!deletedRange) {
-			return;
-		}
-		const deletedText = source.substring(deletedRange.start, deletedRange.endExclusive);
-		for (let line of deletedText.split(/\r\n|\r|\n/)) {
-			line = line.trim();
-			for (const piece of to4grams(line)) {
-				deleted.add(piece);
-			}
-		}
-	};
-	const tryAddInserted = (insertedText: string) => {
-		for (let line of insertedText.split(/\r\n|\r|\n/)) {
-			line = line.trim();
-			for (const piece of to4grams(line)) {
-				inserted.add(piece);
-			}
-		}
-	};
-	for (const e of edit.replacements) {
-		const e1 = e.removeCommonPrefix(source).removeCommonSuffix(source);
-		const e2 = e.removeCommonSuffix(source).removeCommonPrefix(source);
-		if (e1.isEmpty) {
-			continue;
-		}
-		tryAddDeleted(e1.replaceRange);
-		tryAddDeleted(e2.replaceRange);
-		tryAddDeleted(e1.replaceRange.intersect(e2.replaceRange));
-
-		// tryAddInserted(e1.newText);
-		// tryAddInserted(e2.newText);
-		// e1 might have a suffix overlap with the prefix of e1
-		tryAddInserted(trimOverlap(e1.newText, e2.newText));
-	}
-	return new InformationDelta(inserted, deleted);
-}
-
-function trimOverlap(stringToEliminateEnd: string, stringToEliminateStart: string): string {
-	const length = Math.min(stringToEliminateEnd.length, stringToEliminateStart.length);
-	for (let trimLength = 0; trimLength < length; trimLength++) {
-		const str1 = stringToEliminateEnd.slice(0, stringToEliminateEnd.length - trimLength);
-		const str2 = stringToEliminateStart.slice(trimLength);
-		if (str1 === str2) {
-			return str1;
-		}
-	}
-	return '';
-}
-
-function to4grams(text: string) {
-	const result: string[] = [];
-	for (let i = 4; i < text.length; i++) {
-		const ngram = text.slice(i - 4, i);
-		result.push(ngram);
-	}
-	return result;
-}
-
-function setUnion(a: Set<string>, b: Set<string>): Set<string> {
-	const result = new Set<string>();
-	for (const el of a) {
-		result.add(el);
-	}
-	for (const el of b) {
-		result.add(el);
-	}
-	return result;
-}
-
-function setMinus(a: Set<string>, b: Set<string>): Set<string> {
-	const result = new Set<string>();
-	for (const el of a) {
-		if (!b.has(el)) {
-			result.add(el);
-		}
-	}
-	return result;
-}
-
-function setIntersectionCount(a: Set<string>, b: Set<string>): number {
-	let result = 0;
-	for (const el of a) {
-		if (b.has(el)) {
-			result++;
-		}
-	}
-	return result;
-}