@type-editor/changeset 0.0.2

package/src/Span.ts

@@ -0,0 +1,170 @@
+import {isUndefinedOrNull} from '@type-editor/commons';
+
+
+/**
+ * Stores metadata for a part of a change.
+ *
+ * A Span represents a contiguous range in a document with associated metadata.
+ * Spans are immutable and can be sliced, joined, or cut to create new spans.
+ *
+ * @template Data - The type of metadata associated with this span.
+ */
+export class Span<Data = any> {
+
+    /** An empty span array constant to avoid allocations. */
+    public static readonly none: ReadonlyArray<Span> = [];
+
+    private readonly _length: number;
+    private readonly _data: Data;
+
+    /**
+     * Creates a new Span with the specified length and associated data.
+     *
+     * @param length - The length of this span (must be non-negative).
+     * @param data - The metadata associated with this span.
+     */
+    constructor (length: number, data: Data) {
+        this._length = length;
+        this._data = data;
+    }
+
+    /** Returns the length of this span. */
+    get length(): number {
+        return this._length;
+    }
+
+    /** Returns the metadata associated with this span. */
+    get data(): Data {
+        return this._data;
+    }
+
+    /**
+     * Creates a new span with a different length but the same data.
+     * If the length matches, returns this span to avoid allocation.
+     *
+     * @param length - The length for the new span.
+     * @returns A span with the specified length.
+     */
+    private cut(length: number): Span<Data> {
+        return length === this.length ? this : new Span(length, this._data);
+    }
+
+
+    /**
+     * Slices a range from an array of spans.
+     *
+     * This extracts a sub-range from a span array, potentially cutting spans
+     * that partially overlap with the range boundaries.
+     *
+     * @template Data - The type of metadata in the spans.
+     * @param spans - The array of spans to slice from.
+     * @param from - The start position (inclusive).
+     * @param to - The end position (exclusive).
+     * @returns A new array containing the spans within the specified range.
+     */
+    public static slice<Data>(spans: ReadonlyArray<Span<Data>>,
+                              from: number,
+                              to: number): ReadonlyArray<Span<Data>> {
+        // Empty range
+        if (from === to) {
+            return Span.none;
+        }
+
+        // Fast path: if starting from 0 and spans is empty or has known length
+        // Only check for full range when from === 0 to avoid unnecessary iteration
+        if (from === 0) {
+            // Calculate length only when there's potential for a full-range optimization
+            const totalLength = Span.len(spans);
+            if (to >= totalLength) {
+                return spans;
+            }
+        }
+
+        const result: Array<Span<Data>> = [];
+        let offset = 0;
+
+        for (let i = 0; offset < to && i < spans.length; i++) {
+            const span: Span<Data> = spans[i];
+            const spanEnd: number = offset + span.length;
+
+            // Calculate how much of this span overlaps with [from, to)
+            const overlapStart: number = Math.max(from, offset);
+            const overlapEnd: number = Math.min(to, spanEnd);
+            const overlapLength: number = overlapEnd - overlapStart;
+
+            if (overlapLength > 0) {
+                result.push(span.cut(overlapLength));
+            }
+
+            offset = spanEnd;
+        }
+
+        return result;
+    }
+
+
+    /**
+     * Joins two span arrays, potentially combining adjacent spans at the boundary.
+     *
+     * When the last span of the first array and the first span of the second array
+     * meet, the combine function is called. If it returns a value, those spans are
+     * merged into a single span with the combined data.
+     *
+     * @template Data - The type of metadata in the spans.
+     * @param spanListA - The first array of spans.
+     * @param spanListB - The second array of spans.
+     * @param combine - Function to combine the data of adjacent spans. Returns
+     *                  the combined data, or null/undefined if spans should not be merged.
+     * @returns A new array containing the joined spans.
+     */
+    public static join<Data>(spanListA: ReadonlyArray<Span<Data>>,
+                             spanListB: ReadonlyArray<Span<Data>>,
+                             combine: (dataA: Data, dataB: Data) => Data): ReadonlyArray<Span<Data>> {
+        // Fast paths for empty arrays
+        if (spanListA.length === 0) {
+            return spanListB;
+        }
+
+        if (spanListB.length === 0) {
+            return spanListA;
+        }
+
+        // Try to combine the boundary spans
+        const lastSpanA: Span<Data> = spanListA[spanListA.length - 1];
+        const firstSpanB: Span<Data> = spanListB[0];
+        const combinedData: Data = combine(lastSpanA.data, firstSpanB.data);
+
+        // If spans cannot be combined, just concatenate
+        if (isUndefinedOrNull(combinedData)) {
+            return spanListA.concat(spanListB);
+        }
+
+        // Build result with merged boundary span
+        const result: Array<Span<Data>> = spanListA.slice(0, spanListA.length - 1);
+        const mergedSpan = new Span(lastSpanA.length + firstSpanB.length, combinedData);
+        result.push(mergedSpan);
+
+        // Add remaining spans from spanListB
+        for (let i = 1; i < spanListB.length; i++) {
+            result.push(spanListB[i]);
+        }
+
+        return result;
+    }
+
+
+    /**
+     * Calculates the total length of an array of spans.
+     *
+     * @template Data - The type of metadata in the spans.
+     * @param spans - The array of spans to measure.
+     * @returns The sum of all span lengths.
+     */
+    private static len<Data>(spans: ReadonlyArray<Span<Data>>): number {
+        let totalLength = 0;
+        for (const span of spans) {
+            totalLength += span.length;
+        }
+        return totalLength;
+    }
+}

package/src/compute-diff.ts

@@ -0,0 +1,100 @@
+import type {Fragment} from '@type-editor/model';
+
+import type {Change} from './Change';
+import {DefaultEncoder} from './default-encoder';
+import {runMyersDiff} from './myers-diff/run-myers-diff';
+import {tokenizeFragment} from './tokenizer/tokenize-fragment';
+import type {TokenEncoder} from './types/TokenEncoder';
+import type {TrimmedRange} from './types/TrimmedRange';
+
+
+/**
+ * Compute the difference between two fragments using Myers' diff algorithm.
+ *
+ * This implementation optimizes by first scanning from both ends to eliminate
+ * unchanged content, then applies the Myers algorithm to the remaining content.
+ * For performance reasons, the diff computation is limited by MAX_DIFF_SIZE.
+ *
+ * @param fragA - The first fragment to compare.
+ * @param fragB - The second fragment to compare.
+ * @param range - The change range to analyze.
+ * @param encoder - The encoder to use for tokenization (defaults to DefaultEncoder).
+ * @returns An array of Change objects representing the differences.
+ */
+export function computeDiff<T>(fragA: Fragment,
+                               fragB: Fragment,
+                               range: Change,
+                               encoder: TokenEncoder<T> = DefaultEncoder as TokenEncoder<T>): Array<Change> {
+    const tokensA: Array<T> = tokenizeFragment(fragA, encoder, range.fromA, range.toA, []);
+    const tokensB: Array<T> = tokenizeFragment(fragB, encoder, range.fromB, range.toB, []);
+
+    // Create a bound comparison function to avoid unbound method issues
+    const compareTokens = (a: T, b: T): boolean => encoder.compareTokens(a, b);
+
+    const trimmedRange: TrimmedRange = trimEqualTokens(tokensA, tokensB, compareTokens);
+
+    if (trimmedRange.start === tokensA.length && trimmedRange.start === tokensB.length) {
+        return [];
+    }
+
+    // If the result is simple or too large to compute efficiently, return the entire range
+    if (isSimpleDiff(trimmedRange)) {
+        return [range.slice(trimmedRange.start, trimmedRange.endA, trimmedRange.start, trimmedRange.endB)];
+    }
+
+    // Apply Myers' diff algorithm to find the optimal diff
+    const changes: Array<Change> = runMyersDiff(
+        tokensA,
+        tokensB,
+        trimmedRange,
+        compareTokens,
+        range
+    );
+
+    return changes ?? [range.slice(trimmedRange.start, trimmedRange.endA, trimmedRange.start, trimmedRange.endB)];
+}
+
+
+/**
+ * Trim equal tokens from both the start and end of the sequences.
+ * This optimization reduces the amount of work needed by the diff algorithm.
+ *
+ * @param tokensA - The first token sequence.
+ * @param tokensB - The second token sequence.
+ * @param compareTokens - The function to compare tokens for equality.
+ * @returns An object containing the trimmed start and end positions.
+ */
+function trimEqualTokens<T>(tokensA: Array<T>,
+                            tokensB: Array<T>,
+                            compareTokens: (a: T, b: T) => boolean): TrimmedRange {
+    let start = 0;
+    let endA = tokensA.length;
+    let endB = tokensB.length;
+
+    // Scan from the start
+    while (start < tokensA.length && start < tokensB.length && compareTokens(tokensA[start], tokensB[start])) {
+        start++;
+    }
+
+    // Scan from the end
+    while (endA > start && endB > start && compareTokens(tokensA[endA - 1], tokensB[endB - 1])) {
+        endA--;
+        endB--;
+    }
+
+    return {start, endA, endB};
+}
+
+/**
+ * Check if the diff is simple enough to return without running the full algorithm.
+ * A diff is considered simple if one or both sequences are empty after trimming,
+ * or if the remaining sequences have only one token each.
+ *
+ * @param range - The trimmed range to check.
+ * @returns True if the diff is simple, false otherwise.
+ */
+function isSimpleDiff(range: TrimmedRange): boolean {
+    return range.endA === range.start
+        || range.endB === range.start
+        || (range.endA === range.endB && range.endA === range.start + 1);
+}

package/src/default-encoder.ts

@@ -0,0 +1,41 @@
+import type {Node, NodeType} from '@type-editor/model';
+
+import type {TokenEncoder} from './types/TokenEncoder';
+
+
+/**
+ * The default token encoder for diff operations.
+ * - Node start tokens are encoded as strings containing the node name
+ * - Characters are encoded as their character code
+ * - Node end tokens are encoded as negative type IDs
+ */
+export const DefaultEncoder: TokenEncoder<number | string> = {
+    encodeCharacter: (char: number): number => char,
+    encodeNodeStart: (node: Node): string => node.type.name,
+    encodeNodeEnd: (node: Node): number => -getTypeID(node.type),
+    compareTokens: (a: number | string, b: number | string): boolean => a === b
+};
+
+/**
+ * Get a unique numeric ID for a node type.
+ * Uses a cached mapping to avoid recomputing IDs for the same type.
+ *
+ * @param type - The node type to get an ID for.
+ * @returns A unique numeric identifier for the node type.
+ */
+function getTypeID(type: NodeType): number {
+    // Initialize or retrieve the cache from the schema
+    let idCache: Record<string, number> = type.schema.cached.changeSetIDs as Record<string, number> | undefined;
+
+    if (!idCache) {
+        // Build the entire ID cache at once to avoid repeated Object.keys() calls
+        idCache = {} as Record<string, number>;
+        const nodeNames: Array<string> = Object.keys(type.schema.nodes);
+        for (let i = 0; i < nodeNames.length; i++) {
+            idCache[nodeNames[i]] = i + 1;
+        }
+        type.schema.cached.changeSetIDs = idCache;
+    }
+
+    return idCache[type.name];
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export {Change} from './Change';
+export {ChangeSet} from './ChangeSet';
+export {computeDiff} from './compute-diff';
+export {DefaultEncoder} from './default-encoder';
+export {simplifyChanges} from './simplify-changes';
+export {Span} from './Span';
+export type {ChangeSetConfig} from './types/ChangeSetConfig';
+export type {TokenEncoder} from './types/TokenEncoder';

package/src/max-simplify-distance.ts

@@ -0,0 +1,7 @@
+
+
+/**
+ * Maximum distance (in characters) between changes for them to be considered
+ * candidates for merging during simplification.
+ */
+export const MAX_SIMPLIFY_DISTANCE = 30;

package/src/myers-diff/run-myers-diff.ts

@@ -0,0 +1,261 @@
+import type {Change} from '../Change';
+import type {TrimmedRange} from '../types/TrimmedRange';
+
+
+/**
+ * State for accumulating changes during traceback.
+ */
+interface ChangeAccumulator {
+    fromA: number;
+    toA: number;
+    fromB: number;
+    toB: number;
+}
+
+/**
+ * The code below will refuse to compute a diff with more than 5000
+ * insertions or deletions, which takes about 300ms to reach on my
+ * machine. This is a safeguard against runaway computations.
+ */
+const MAX_DIFF_SIZE = 5000;
+
+
+/**
+ * Run Myers' diff algorithm to find the optimal sequence of changes.
+ *
+ * Myers' algorithm uses dynamic programming to find the shortest edit script
+ * between two sequences. It explores diagonals in the edit graph, maintaining
+ * a frontier of the furthest reaching paths.
+ *
+ * See: https://neil.fraser.name/writing/diff/myers.pdf
+ * See: https://blog.jcoglan.com/2017/02/12/the-myers-diff-algorithm-part-1/
+ *
+ * @param tokensA - The first token sequence.
+ * @param tokensB - The second token sequence.
+ * @param range - The trimmed range to process.
+ * @param compareTokens - The function to compare tokens for equality.
+ * @param changeRange - The original change range for creating result changes.
+ * @returns An array of changes, or null if the algorithm exceeded MAX_DIFF_SIZE.
+ */
+export function runMyersDiff<T>(tokensA: Array<T>,
+                                tokensB: Array<T>,
+                                range: TrimmedRange,
+                                compareTokens: (a: T, b: T) => boolean,
+                                changeRange: Change): Array<Change> | null {
+    const lengthA: number = range.endA - range.start;
+    const lengthB: number = range.endB - range.start;
+    const maxOperations: number = Math.min(MAX_DIFF_SIZE, lengthA + lengthB);
+    const offset: number = maxOperations + 1;
+    const history: Array<Array<number>> = [];
+    const frontier: Array<number> = initializeFrontier(offset * 2);
+
+    for (let editDistance = 0; editDistance <= maxOperations; editDistance++) {
+        for (let diagonal = -editDistance; diagonal <= editDistance; diagonal += 2) {
+            const found: boolean = exploreDiagonal(
+                tokensA,
+                tokensB,
+                frontier,
+                diagonal,
+                offset,
+                range,
+                lengthA,
+                lengthB,
+                compareTokens
+            );
+
+            if (found) {
+                return tracebackChanges(
+                    history,
+                    frontier,
+                    diagonal,
+                    offset,
+                    editDistance,
+                    range,
+                    changeRange
+                );
+            }
+        }
+
+        // Save frontier state every other iteration (for odd/even diagonal alternation)
+        if (editDistance % 2 === 0) {
+            history.push(frontier.slice());
+        }
+    }
+
+    // Algorithm exceeded maximum complexity
+    return null;
+}
+
+/**
+ * Initialize the frontier array with -1 values.
+ *
+ * @param size - The size of the frontier array.
+ * @returns The initialized frontier array.
+ */
+function initializeFrontier(size: number): Array<number> {
+    return new Array<number>(size).fill(-1);
+}
+
+/**
+ * Explore a diagonal in the edit graph, extending as far as possible with matching tokens.
+ *
+ * @param tokensA - The first token sequence.
+ * @param tokensB - The second token sequence.
+ * @param frontier - The current frontier array.
+ * @param diagonal - The diagonal to explore.
+ * @param offset - The offset for indexing the frontier.
+ * @param range - The trimmed range.
+ * @param lengthA - The length of sequence A.
+ * @param lengthB - The length of sequence B.
+ * @param compareTokens - The function to compare tokens.
+ * @returns True if a complete path was found, false otherwise.
+ */
+function exploreDiagonal<T>(tokensA: Array<T>,
+                            tokensB: Array<T>,
+                            frontier: Array<number>,
+                            diagonal: number,
+                            offset: number,
+                            range: TrimmedRange,
+                            lengthA: number,
+                            lengthB: number,
+                            compareTokens: (a: T, b: T) => boolean): boolean {
+    const nextValue: number = frontier[diagonal + 1 + offset];
+    const prevValue: number = frontier[diagonal - 1 + offset];
+
+    // Choose the path that goes furthest
+    let x: number = nextValue < prevValue ? prevValue : nextValue + 1;
+    let y: number = x + diagonal;
+
+    // Pre-compute start offset to avoid repeated addition in the loop
+    const startOffset: number = range.start;
+
+    // Extend along the diagonal as far as possible with matching tokens
+    while (x < lengthA && y < lengthB && compareTokens(tokensA[startOffset + x], tokensB[startOffset + y])) {
+        x++;
+        y++;
+    }
+
+    frontier[diagonal + offset] = x;
+
+    // Check if we've reached the end of both sequences
+    return x >= lengthA && y >= lengthB;
+}
+
+/**
+ * Trace back through the history to build the set of changes.
+ *
+ * @param history - The history of frontier states.
+ * @param frontier - The final frontier state.
+ * @param diagonal - The final diagonal position.
+ * @param offset - The offset for indexing.
+ * @param editDistance - The final edit distance.
+ * @param range - The trimmed range.
+ * @param changeRange - The original change range.
+ * @returns An array of changes in forward order.
+ */
+function tracebackChanges(history: Array<Array<number>>,
+                          frontier: Array<number>,
+                          diagonal: number,
+                          offset: number,
+                          editDistance: number,
+                          range: TrimmedRange,
+                          changeRange: Change): Array<Change> {
+    const changes: Array<Change> = [];
+    const minSpan: number = computeMinUnchangedThreshold(range.endA - range.start, range.endB - range.start);
+
+    // Accumulator for building changes, initialized to "no change pending"
+    const accumulator: ChangeAccumulator = {
+        fromA: -1,
+        toA: -1,
+        fromB: -1,
+        toB: -1
+    };
+
+    let currentDiagonal: number = diagonal;
+    let currentFrontier: Array<number> = frontier;
+
+    for (let i = editDistance - 1; i >= 0; i--) {
+        const nextValue: number = currentFrontier[currentDiagonal + 1 + offset];
+        const prevValue: number = currentFrontier[currentDiagonal - 1 + offset];
+
+        let x: number;
+        let y: number;
+
+        if (nextValue < prevValue) {
+            // Deletion
+            currentDiagonal--;
+            x = prevValue + range.start;
+            y = x + currentDiagonal;
+            addChange(accumulator, changes, changeRange, minSpan, x, x, y, y + 1);
+        } else {
+            // Insertion
+            currentDiagonal++;
+            x = nextValue + range.start;
+            y = x + currentDiagonal;
+            addChange(accumulator, changes, changeRange, minSpan, x, x + 1, y, y);
+        }
+
+        currentFrontier = history[i >> 1];
+    }
+
+    // Flush any pending change
+    if (accumulator.fromA > -1) {
+        changes.push(changeRange.slice(accumulator.fromA, accumulator.toA, accumulator.fromB, accumulator.toB));
+    }
+
+    return changes.reverse();
+}
+
+/**
+ * Compute the minimum length of an unchanged range (not at the start/end of the compared content).
+ *
+ * The algorithm scales the threshold based on the size of the input, which prevents
+ * the diff from becoming fragmented with coincidentally matching characters when
+ * comparing larger texts. For small diffs, the threshold is 2; for larger ones,
+ * it scales up to a maximum of 15.
+ *
+ * @param sizeA - The size of the first sequence.
+ * @param sizeB - The size of the second sequence.
+ * @returns The minimum length for an unchanged range (between 2 and 15).
+ */
+function computeMinUnchangedThreshold(sizeA: number, sizeB: number): number {
+    const maxSize = Math.max(sizeA, sizeB);
+    const scaledThreshold = Math.floor(maxSize / 10);
+    return Math.min(15, Math.max(2, scaledThreshold));
+}
+
+/**
+ * Add a change to the accumulator, merging with the previous change if they're close enough.
+ *
+ * @param accumulator - The change accumulator state.
+ * @param changes - The array of changes being built.
+ * @param changeRange - The original change range.
+ * @param minSpan - The minimum span between changes to keep them separate.
+ * @param fromA - The start position in sequence A.
+ * @param toA - The end position in sequence A.
+ * @param fromB - The start position in sequence B.
+ * @param toB - The end position in sequence B.
+ */
+function addChange(accumulator: ChangeAccumulator,
+                   changes: Array<Change>,
+                   changeRange: Change,
+                   minSpan: number,
+                   fromA: number,
+                   toA: number,
+                   fromB: number,
+                   toB: number): void {
+    if (accumulator.fromA > -1 && accumulator.fromA < toA + minSpan) {
+        // Merge with the existing accumulated change
+        accumulator.fromA = fromA;
+        accumulator.fromB = fromB;
+    } else {
+        // Flush the previous change and start a new one
+        if (accumulator.fromA > -1) {
+            changes.push(changeRange.slice(accumulator.fromA, accumulator.toA, accumulator.fromB, accumulator.toB));
+        }
+        accumulator.fromA = fromA;
+        accumulator.toA = toA;
+        accumulator.fromB = fromB;
+        accumulator.toB = toB;
+    }
+}

package/src/simplify-changes/expand-to-word-boundaries.ts

@@ -0,0 +1,43 @@
+import {isLetter} from './is-letter';
+
+
+/**
+ * Expands a position range to word boundaries.
+ *
+ * If the range starts or ends within a word, expands it to include the entire word.
+ * This ensures that changes don't split words in confusing ways.
+ *
+ * @param text - The text to analyze.
+ * @param textStart - The offset where the analyzed text starts in the document.
+ * @param textEnd - The end of the text context.
+ * @param fromPos - The start position to expand.
+ * @param toPos - The end position to expand.
+ * @returns A tuple with the expanded [from, to] positions.
+ */
+export function expandToWordBoundaries(text: string,
+                                       textStart: number,
+                                       textEnd: number,
+                                       fromPos: number,
+                                       toPos: number): [number, number] {
+    let expandedFrom: number = fromPos;
+    let expandedTo: number = toPos;
+
+    // Expand start position backward to word boundary
+    if (expandedFrom < textEnd && isLetter(text.charCodeAt(expandedFrom - textStart))) {
+        while (expandedFrom > textStart && isLetter(text.charCodeAt(expandedFrom - 1 - textStart))) {
+            expandedFrom--;
+        }
+    }
+
+    // Expand end position forward to word boundary
+    if (expandedTo > textStart
+        && expandedTo <= textEnd
+        && isLetter(text.charCodeAt(expandedTo - 1 - textStart))) {
+
+        while (expandedTo < textEnd && isLetter(text.charCodeAt(expandedTo - textStart))) {
+            expandedTo++;
+        }
+    }
+
+    return [expandedFrom, expandedTo];
+}