@type-editor/changeset 0.0.2

package/src/simplify-changes/fill-change.ts

@@ -0,0 +1,99 @@
+import {Change} from '../Change';
+import {Span} from '../Span';
+
+
+/**
+ * Creates a merged change that spans from fromB to toB, filling gaps between changes.
+ *
+ * This function takes a sequence of changes and creates a single change that covers
+ * a broader range, filling any gaps between the original changes. The gaps are filled
+ * with spans using the data from adjacent changes.
+ *
+ * @param changes - The array of changes to merge.
+ * @param fromB - The start position in the new document.
+ * @param toB - The end position in the new document.
+ * @returns A new Change covering the expanded range.
+ */
+export function fillChange(changes: ReadonlyArray<Change>, fromB: number, toB: number): Change {
+    // Calculate the corresponding positions in the old document
+    const fromA: number = changes[0].fromA - (changes[0].fromB - fromB);
+    const lastChange: Change = changes[changes.length - 1];
+    const toA: number = lastChange.toA + (toB - lastChange.toB);
+
+    // Initialize span arrays
+    let deletedSpans: ReadonlyArray<Span> = Span.none;
+    let insertedSpans: ReadonlyArray<Span> = Span.none;
+
+    // Get initial data for filling gaps (prefer actual changes over empty ones)
+    const initialSpans = changes[0].deleted.length ? changes[0].deleted :
+        changes[0].inserted.length ? changes[0].inserted : null;
+    let deletedData = initialSpans?.[0]?.data;
+    let insertedData = (changes[0].inserted.length
+        ? changes[0].inserted
+        : changes[0].deleted.length ? changes[0].deleted : null)?.[0]?.data;
+
+    let positionA = fromA;
+    let positionB = fromB;
+
+    for (let i = 0; i <= changes.length; i++) {
+        const currentChange: Change | null = i === changes.length ? null : changes[i];
+        const endA: number = currentChange ? currentChange.fromA : toA;
+        const endB: number = currentChange ? currentChange.fromB : toB;
+
+        // Fill gap in old document if there is one
+        if (endA > positionA) {
+            deletedSpans = Span.join(
+                deletedSpans,
+                [new Span(endA - positionA, deletedData)],
+                combineSpanData
+            );
+        }
+
+        // Fill gap in new document if there is one
+        if (endB > positionB) {
+            insertedSpans = Span.join(
+                insertedSpans,
+                [new Span(endB - positionB, insertedData)],
+                combineSpanData
+            );
+        }
+
+        // If we've processed all changes, we're done
+        if (!currentChange) {
+            break;
+        }
+
+        // Add the actual change spans
+        deletedSpans = Span.join(deletedSpans, currentChange.deleted, combineSpanData);
+        insertedSpans = Span.join(insertedSpans, currentChange.inserted, combineSpanData);
+
+        // Update data for the next gap fill
+        if (deletedSpans.length > 0) {
+            deletedData = deletedSpans[deletedSpans.length - 1].data;
+        }
+        if (insertedSpans.length > 0) {
+            insertedData = insertedSpans[insertedSpans.length - 1].data;
+        }
+
+        // Move positions forward
+        positionA = currentChange.toA;
+        positionB = currentChange.toB;
+    }
+
+    return new Change(fromA, toA, fromB, toB, deletedSpans, insertedSpans);
+}
+
+/**
+ * Combines two span data values by returning the value if they're equal, or null otherwise.
+ *
+ * This function is used as a callback for Span.join operations to determine if adjacent
+ * spans with the same data can be merged.
+ *
+ * @template T - The type of data being compared.
+ * @param a - The first data value.
+ * @param b - The second data value.
+ * @returns The data value if both are equal, null otherwise.
+ */
+function combineSpanData<T>(a: T, b: T): T | null {
+    return a === b ? a : null;
+}

package/src/simplify-changes/get-text.ts

@@ -0,0 +1,69 @@
+import type {Fragment, Node} from '@type-editor/model';
+
+
+/**
+ * Extracts text content from a document fragment range.
+ *
+ * Converts a range of document nodes into a string representation for
+ * character-level analysis. Non-text elements (images, widgets, etc.) are
+ * represented as spaces to prevent them from being considered part of words.
+ *
+ * @param fragment - The document fragment to extract text from.
+ * @param start - The start position of the range (inclusive).
+ * @param end - The end position of the range (exclusive).
+ * @returns The text content with non-text nodes replaced by spaces.
+ */
+export function getText(fragment: Fragment, start: number, end: number): string {
+    const resultParts: Array<string> = [];
+
+    /**
+     * Recursively extracts text from a fragment, handling different node types.
+     *
+     * @param frag - The fragment to process.
+     * @param rangeStart - The start position relative to the fragment.
+     * @param rangeEnd - The end position relative to the fragment.
+     */
+    function extractText(frag: Fragment, rangeStart: number, rangeEnd: number): void {
+        let offset = 0;
+
+        for (let i = 0; i < frag.childCount; i++) {
+            const child: Node = frag.child(i);
+            const childEnd: number = offset + child.nodeSize;
+            const overlapStart: number = Math.max(offset, rangeStart);
+            const overlapEnd: number = Math.min(childEnd, rangeEnd);
+
+            // Only process nodes that overlap with our range
+            if (overlapStart < overlapEnd) {
+                if (child.isText) {
+                    // Extract the relevant portion of text content
+                    const textStart: number = Math.max(0, rangeStart - offset);
+                    const textEnd: number = Math.min(child.text.length, rangeEnd - offset);
+                    resultParts.push(child.text.slice(textStart, textEnd));
+                } else if (child.isLeaf) {
+                    // Leaf nodes (like images) are represented as spaces
+                    resultParts.push(' ');
+                } else {
+                    // Non-leaf block nodes: add space before if at start
+                    if (overlapStart === offset) {
+                        resultParts.push(' ');
+                    }
+
+                    // Recursively process the node's content
+                    const contentStart: number = Math.max(0, overlapStart - offset - 1);
+                    const contentEnd: number = Math.min(child.content.size, overlapEnd - offset);
+                    extractText(child.content, contentStart, contentEnd);
+
+                    // Add space after if at end
+                    if (overlapEnd === childEnd) {
+                        resultParts.push(' ');
+                    }
+                }
+            }
+
+            offset = childEnd;
+        }
+    }
+
+    extractText(fragment, start, end);
+    return resultParts.join('');
+}

package/src/simplify-changes/has-word-boundary.ts

@@ -0,0 +1,34 @@
+import {isLetter} from './is-letter';
+
+
+/**
+ * Checks if there's a word boundary between two changes.
+ *
+ * A word boundary is detected when there's a transition from non-letter to letter
+ * or vice versa between the end of one change and the start of the next.
+ *
+ * @param text - The text to analyze.
+ * @param textStart - The offset where the analyzed text starts in the document.
+ * @param fromPos - The start position to check from.
+ * @param toPos - The end position to check to.
+ * @param contextEnd - The end of the text context (to avoid out-of-bounds).
+ * @returns True if a word boundary is found, false otherwise.
+ */
+export function hasWordBoundary(text: string,
+                                textStart: number,
+                                fromPos: number,
+                                toPos: number,
+                                contextEnd: number): boolean {
+    // Check each position in the range for non-letter characters
+    for (let pos = fromPos; pos < toPos; pos++) {
+        const isLetterChar: boolean = pos >= contextEnd ? false : isLetter(text.charCodeAt(pos - textStart));
+
+        // A word boundary exists if any character in the range is not a letter
+        if (!isLetterChar) {
+            return true;
+        }
+    }
+
+    return false;
+}
+

package/src/simplify-changes/is-letter.ts

@@ -0,0 +1,62 @@
+
+/** Unicode letter detection regex (if supported by the runtime). */
+let unicodeLetterRegex: RegExp | undefined;
+
+/**
+ * Initialize Unicode letter detection regex if the runtime supports it.
+ * Falls back to alternative detection methods if not available.
+ */
+try {
+    unicodeLetterRegex = new RegExp('[\\p{Alphabetic}_]', 'u');
+} catch (_) {
+    // Unicode property escapes not supported
+}
+
+
+/**
+ * Regular expression matching common single-case script characters.
+ * These characters are always word characters but don't have distinct
+ * upper/lowercase forms (Hebrew, Arabic, CJK, etc.).
+ */
+const nonASCIISingleCaseWordChar = /[\u00df\u0587\u0590-\u05f4\u0600-\u06ff\u3040-\u309f\u30a0-\u30ff\u3400-\u4db5\u4e00-\u9fcc\uac00-\ud7af]/;
+
+
+/**
+ * ASCII character code ranges for word characters.
+ */
+const ASCII_DIGIT_START = 48;    // '0'
+const ASCII_DIGIT_END = 57;      // '9'
+const ASCII_UPPER_START = 65;    // 'A'
+const ASCII_UPPER_END = 90;      // 'Z'
+const ASCII_LOWER_START = 97;    // 'a'
+const ASCII_LOWER_END = 122;     // 'z'
+
+
+/**
+ * Determines whether a character code represents a letter or digit.
+ *
+ * For ASCII characters, checks if the code is in the alphanumeric range.
+ * For non-ASCII characters, uses Unicode properties if available, otherwise
+ * checks for case changes or single-case script membership.
+ *
+ * @param code - The character code to test.
+ * @returns True if the character is a letter or digit, false otherwise.
+ */
+export function isLetter(code: number): boolean {
+    // Fast path for ASCII characters
+    if (code < 128) {
+        return (code >= ASCII_DIGIT_START && code <= ASCII_DIGIT_END)
+            || (code >= ASCII_UPPER_START && code <= ASCII_UPPER_END)
+            || (code >= ASCII_LOWER_START && code <= ASCII_LOWER_END);
+    }
+
+    const char = String.fromCharCode(code);
+
+    // Use Unicode properties if available
+    if (unicodeLetterRegex) {
+        return unicodeLetterRegex.test(char);
+    }
+
+    // Fallback: check for case changes or single-case script membership
+    return char.toUpperCase() !== char.toLowerCase() || nonASCIISingleCaseWordChar.test(char);
+}

package/src/simplify-changes/simplify-adjacent-changes.ts

@@ -0,0 +1,111 @@
+import type {Node} from '@type-editor/model';
+
+import {Change} from '../Change';
+import {MAX_SIMPLIFY_DISTANCE} from '../max-simplify-distance';
+import {expandToWordBoundaries} from './expand-to-word-boundaries';
+import {fillChange} from './fill-change';
+import {getText} from './get-text';
+import {hasWordBoundary} from './has-word-boundary';
+
+
+/**
+ * Processes a group of adjacent changes and adds simplified versions to the target array.
+ *
+ * This function examines changes in a group to determine if they should be merged.
+ * Changes are merged if they're within the same word (no word boundary between them).
+ * Mixed insertions/deletions are expanded to word boundaries unless they're single
+ * character replacements.
+ *
+ * @param changes - The complete array of changes.
+ * @param from - The start index in the changes array (inclusive).
+ * @param to - The end index in the changes array (exclusive).
+ * @param doc - The document node to analyze.
+ * @param target - The array to add simplified changes to.
+ */
+export function simplifyAdjacentChanges(changes: ReadonlyArray<Change>,
+                                        from: number,
+                                        to: number,
+                                        doc: Node,
+                                        target: Array<Change>): void {
+    // Get text context around the changes for word boundary detection
+    const contextStart: number = Math.max(0, changes[from].fromB - MAX_SIMPLIFY_DISTANCE);
+    const contextEnd: number = Math.min(doc.content.size, changes[to - 1].toB + MAX_SIMPLIFY_DISTANCE);
+    const text: string = getText(doc.content, contextStart, contextEnd);
+
+    for (let i = from; i < to; i++) {
+        const groupStartIndex: number = i;
+        let lastChange: Change = changes[i];
+        let totalDeleted: number = lastChange.lenA;
+        let totalInserted: number = lastChange.lenB;
+
+        // Try to merge consecutive changes that are within the same word
+        while (i < to - 1) {
+            const nextChange: Change = changes[i + 1];
+
+            // Check if there's a word boundary between the current and next change
+            const hasWindowBoundary: boolean = hasWordBoundary(
+                text,
+                contextStart,
+                lastChange.toB,
+                nextChange.fromB,
+                contextEnd
+            );
+
+            if (hasWindowBoundary) {
+                break;
+            }
+
+            // Accumulate the change metrics
+            totalDeleted += nextChange.lenA;
+            totalInserted += nextChange.lenB;
+            lastChange = nextChange;
+            i++;
+        }
+
+        // Determine if we should expand this change group to word boundaries
+        const shouldExpandToWords: boolean =
+            totalInserted > 0
+            && totalDeleted > 0
+            && !(totalInserted === 1 && totalDeleted === 1);
+
+        if (shouldExpandToWords) {
+            // Expand the range to word boundaries
+            const [expandedFrom, expandedTo] = expandToWordBoundaries(
+                text,
+                contextStart,
+                contextEnd,
+                changes[groupStartIndex].fromB,
+                changes[i].toB
+            );
+
+            // Create a merged change covering the expanded range
+            const mergedChange: Change = fillChange(
+                changes.slice(groupStartIndex, i + 1),
+                expandedFrom,
+                expandedTo
+            );
+
+            // Try to merge with the previous change in target if they're adjacent
+            const previousChange: Change | null = target.length > 0 ? target[target.length - 1] : null;
+
+            if (previousChange?.toA === mergedChange.fromA) {
+                // Merge with previous change
+                target[target.length - 1] = new Change(
+                    previousChange.fromA,
+                    mergedChange.toA,
+                    previousChange.fromB,
+                    mergedChange.toB,
+                    previousChange.deleted.concat(mergedChange.deleted),
+                    previousChange.inserted.concat(mergedChange.inserted)
+                );
+            } else {
+                target.push(mergedChange);
+            }
+        } else {
+            // Add changes individually without expansion
+            for (let j = groupStartIndex; j <= i; j++) {
+                target.push(changes[j]);
+            }
+        }
+    }
+}

package/src/simplify-changes.ts

@@ -0,0 +1,42 @@
+import type {Node} from '@type-editor/model';
+
+import type {Change} from './Change';
+import {MAX_SIMPLIFY_DISTANCE} from './max-simplify-distance';
+import {simplifyAdjacentChanges} from './simplify-changes/simplify-adjacent-changes';
+
+
+/**
+ * Simplifies a set of changes for presentation.
+ *
+ * This function makes changes more readable by expanding insertions and deletions
+ * that occur within the same word to cover entire words. This prevents confusing
+ * partial-word changes while maintaining accuracy.
+ *
+ * The algorithm:
+ * 1. Groups nearby changes (within MAX_SIMPLIFY_DISTANCE)
+ * 2. For mixed insertions/deletions in a group, expands to word boundaries
+ * 3. Preserves single-character replacements as-is
+ * 4. Merges adjacent changes when appropriate
+ *
+ * @param changes - The array of changes to simplify.
+ * @param doc - The document node (new version) to analyze.
+ * @returns A new array of simplified changes.
+ */
+export function simplifyChanges(changes: ReadonlyArray<Change>, doc: Node): Array<Change> {
+    const result: Array<Change> = [];
+
+    for (let i = 0; i < changes.length; i++) {
+        const groupStart: number = i;
+        let groupEnd: number = changes[i].toB;
+
+        // Group adjacent changes that are within MAX_SIMPLIFY_DISTANCE
+        while (i < changes.length - 1 && changes[i + 1].fromB <= groupEnd + MAX_SIMPLIFY_DISTANCE) {
+            groupEnd = changes[++i].toB;
+        }
+
+        // Process each group of adjacent changes
+        simplifyAdjacentChanges(changes, groupStart, i + 1, doc, result);
+    }
+
+    return result;
+}

package/src/tokenizer/tokenize-block-node.ts

@@ -0,0 +1,47 @@
+import type {Node} from '@type-editor/model';
+
+import type {TokenEncoder} from '../types/TokenEncoder';
+import {tokenizeFragment} from './tokenize-fragment';
+
+
+/**
+ * Tokenize a block (non-leaf) node by encoding its boundaries and recursively
+ * tokenizing its content.
+ *
+ * @param blockNode - The block node to tokenize.
+ * @param encoder - The encoder to use for converting nodes to tokens.
+ * @param rangeStart - The start position in the document.
+ * @param rangeEnd - The end position in the document.
+ * @param nodeOffset - The offset of this node in the document.
+ * @param nodeEndOffset - The end offset of this node in the document.
+ * @param target - The array to append tokens to.
+ */
+export function tokenizeBlockNode<T>(blockNode: Node,
+                                     encoder: TokenEncoder<T>,
+                                     rangeStart: number,
+                                     rangeEnd: number,
+                                     nodeOffset: number,
+                                     nodeEndOffset: number,
+                                     target: Array<T>): void {
+    // Add start token if we're at the beginning of the node
+    if (rangeStart === nodeOffset) {
+        target.push(encoder.encodeNodeStart(blockNode));
+    }
+
+    // Recursively tokenize the node's content
+    const contentStart = Math.max(nodeOffset + 1, rangeStart) - nodeOffset - 1;
+    const contentEnd = Math.min(nodeEndOffset - 1, rangeEnd) - nodeOffset - 1;
+
+    tokenizeFragment(
+        blockNode.content,
+        encoder,
+        contentStart,
+        contentEnd,
+        target
+    );
+
+    // Add end token if we're at the end of the node
+    if (rangeEnd === nodeEndOffset) {
+        target.push(encoder.encodeNodeEnd(blockNode));
+    }
+}

package/src/tokenizer/tokenize-fragment.ts

@@ -0,0 +1,46 @@
+import type {Fragment, Node} from '@type-editor/model';
+
+import type {TokenEncoder} from '../types/TokenEncoder';
+import {tokenizeBlockNode} from './tokenize-block-node';
+import {tokenizeTextNode} from './tokenize-textNode';
+
+
+/**
+ * Convert the given range of a fragment to tokens for diff comparison.
+ * Recursively processes the fragment tree, encoding text characters and node boundaries.
+ *
+ * @param fragment - The fragment to tokenize.
+ * @param encoder - The encoder to use for converting nodes and characters to tokens.
+ * @param start - The start offset within the fragment.
+ * @param end - The end offset within the fragment.
+ * @param target - The array to append tokens to.
+ * @returns The target array with all tokens appended.
+ */
+export function tokenizeFragment<T>(fragment: Fragment,
+                                    encoder: TokenEncoder<T>,
+                                    start: number,
+                                    end: number,
+                                    target: Array<T>): Array<T> {
+    let currentOffset = 0;
+
+    for (let i = 0; i < fragment.childCount; i++) {
+        const child: Node = fragment.child(i);
+        const childEndOffset: number = currentOffset + child.nodeSize;
+        const rangeStart: number = Math.max(currentOffset, start);
+        const rangeEnd: number = Math.min(childEndOffset, end);
+
+        if (rangeStart < rangeEnd) {
+            if (child.isText) {
+                tokenizeTextNode(child, encoder, rangeStart, rangeEnd, currentOffset, target);
+            } else if (child.isLeaf) {
+                target.push(encoder.encodeNodeStart(child));
+            } else {
+                tokenizeBlockNode(child, encoder, rangeStart, rangeEnd, currentOffset, childEndOffset, target);
+            }
+        }
+
+        currentOffset = childEndOffset;
+    }
+
+    return target;
+}

package/src/tokenizer/tokenize-textNode.ts

@@ -0,0 +1,31 @@
+import type {Node} from '@type-editor/model';
+
+import type {TokenEncoder} from '../types/TokenEncoder';
+
+
+/**
+ * Tokenize a text node by encoding each character within the specified range.
+ *
+ * @param textNode - The text node to tokenize.
+ * @param encoder - The encoder to use for converting characters to tokens.
+ * @param rangeStart - The start position in the document.
+ * @param rangeEnd - The end position in the document.
+ * @param nodeOffset - The offset of this node in the document.
+ * @param target - The array to append tokens to.
+ */
+export function tokenizeTextNode<T>(textNode: Node,
+                                    encoder: TokenEncoder<T>,
+                                    rangeStart: number,
+                                    rangeEnd: number,
+                                    nodeOffset: number,
+                                    target: Array<T>): void {
+    const text = textNode.text;
+    if (!text) {
+        return;
+    }
+
+    for (let j = rangeStart; j < rangeEnd; j++) {
+        const charCode: number = text.charCodeAt(j - nodeOffset);
+        target.push(encoder.encodeCharacter(charCode, textNode.marks));
+    }
+}

package/src/types/ChangeJSON.ts

@@ -0,0 +1,23 @@
+
+/**
+ * JSON-serializable representation of a Change.
+ *
+ * Describes a change between two document versions (A and B), including
+ * the affected ranges and the deleted/inserted content spans.
+ *
+ * @template Data - The type of metadata associated with each span.
+ */
+export interface ChangeJSON<Data> {
+    /** The start position in document A where the change begins. */
+    fromA: number;
+    /** The end position in document A where the change ends. */
+    toA: number;
+    /** The start position in document B where the change begins. */
+    fromB: number;
+    /** The end position in document B where the change ends. */
+    toB: number;
+    /** The spans that were deleted from document A. */
+    deleted: ReadonlyArray<{length: number, data: Data}>;
+    /** The spans that were inserted into document B. */
+    inserted: ReadonlyArray<{length: number, data: Data}>;
+}

package/src/types/ChangeSetConfig.ts

@@ -0,0 +1,19 @@
+import type {Node} from '@type-editor/model';
+
+import type {TokenEncoder} from './TokenEncoder';
+
+
+/**
+ * Configuration options for a ChangeSet.
+ *
+ * @template Data - The type of metadata associated with changes.
+ */
+export interface ChangeSetConfig<Data> {
+    /** The starting document that changes are tracked from. */
+    doc: Node;
+    /** Function to combine metadata from adjacent spans. Returns null if incompatible. */
+    combine: (dataA: Data, dataB: Data) => Data;
+    /** Encoder for tokenizing document content during diff operations. */
+    encoder: TokenEncoder<any>;
+    
+}

package/src/types/TokenEncoder.ts

@@ -0,0 +1,52 @@
+import type {Mark, Node} from '@type-editor/model';
+
+
+/**
+ * A token encoder can be passed when creating a `ChangeSet` in order
+ * to influence the way the library runs its diffing algorithm. The
+ * encoder determines how document tokens (such as nodes and
+ * characters) are encoded and compared.
+ *
+ * Note that both the encoding and the comparison may run a lot, and
+ * doing non-trivial work in these functions could impact
+ * performance.
+ */
+export interface TokenEncoder<T> {
+
+    /**
+     * Encode a given character, with the given marks applied.
+     *
+     * @param char - The character code to encode.
+     * @param marks - The marks applied to the character.
+     * @returns The encoded representation of the character.
+     */
+    encodeCharacter(char: number, marks: ReadonlyArray<Mark>): T;
+
+    /**
+     * Encode the start of a node or, if this is a leaf node, the
+     * entire node.
+     *
+     * @param node - The node to encode.
+     * @returns The encoded representation of the node start.
+     */
+    encodeNodeStart(node: Node): T;
+
+    /**
+     * Encode the end token for the given node. It is valid to encode
+     * every end token in the same way.
+     *
+     * @param node - The node to encode the end token for.
+     * @returns The encoded representation of the node end.
+     */
+    encodeNodeEnd(node: Node): T;
+
+    /**
+     * Compare the given tokens. Should return true when they count as
+     * equal.
+     *
+     * @param a - The first token to compare.
+     * @param b - The second token to compare.
+     * @returns True if the tokens are equal, false otherwise.
+     */
+    compareTokens(a: T, b: T): boolean;
+}

package/src/types/TrimmedRange.ts

@@ -0,0 +1,10 @@
+
+
+/**
+ * Represents a trimmed range after eliminating equal tokens from both ends.
+ */
+export interface TrimmedRange {
+    start: number;
+    endA: number;
+    endB: number;
+}