@ansonlai/docx-redline-js 0.8.1 → 0.8.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ansonlai/docx-redline-js",
-  "version": "0.8.1",
+  "version": "0.8.3",
   "description": "Host-independent OOXML reconciliation engine for .docx manipulation with track changes",
   "license": "MIT",
   "type": "module",

package/pipeline/diff-engine.js

@@ -4,105 +4,121 @@
  * Word-level diffing with offset tracking for precise run splitting.
  */
 
-import { diff_match_patch } from 'diff-match-patch';
-import { DiffOp } from '../core/types.js';
-
-export const TOKEN_CODE_POINT_BASE = 0x10000;
-export const MAX_DIFF_TOKENS = 0x40000;
-
-// diff-match-patch's JavaScript port operates on UTF-16 code units, not
-// Unicode code points. Keep its private encoding in the BMP while skipping
-// surrogates; larger token sets take the deterministic token-array fallback.
-const BMP_FIRST_CODE = 1;
-const BMP_BEFORE_SURROGATES = 0xD800 - BMP_FIRST_CODE;
-const BMP_AFTER_SURROGATES = 0x10000 - 0xE000;
-const DMP_SAFE_TOKEN_LIMIT = BMP_BEFORE_SURROGATES + BMP_AFTER_SURROGATES;
-
-export class DiffTokenLimitError extends Error {
-    constructor(limit = MAX_DIFF_TOKENS) {
-        super(`Word diff exceeds the safe limit of ${limit} unique tokens.`);
-        this.name = 'DiffTokenLimitError';
-        this.code = 'DIFF_TOKEN_LIMIT';
-        this.limit = limit;
-    }
-}
-
-export function isDiffTokenLimitError(error) {
-    return error?.code === 'DIFF_TOKEN_LIMIT';
-}
-
-export function createDiffEngine(options = {}) {
-    const timeout = options.diffTimeoutSeconds ?? 0;
-    if (!Number.isFinite(timeout) || timeout < 0) {
-        throw new TypeError('diffTimeoutSeconds must be a finite non-negative number.');
-    }
-    const engine = new diff_match_patch();
-    engine.Diff_Timeout = timeout;
-    return engine;
-}
-
-function tokenize(text) {
-    const tokens = [];
-    const leading = text.match(/^\s+/);
-    if (leading) tokens.push(leading[0]);
-
-    const regex = /(\S+)(\s*)/g;
-    regex.lastIndex = leading?.[0].length || 0;
-    let match;
-    while ((match = regex.exec(text)) !== null) {
-        if (match[1]) tokens.push(match[1]);
-        if (match[2]) tokens.push(match[2]);
-    }
-    return tokens;
-}
-
+import { diff_match_patch } from 'diff-match-patch';
+import { DiffOp } from '../core/types.js';
+
+export const TOKEN_CODE_POINT_BASE = 0x10000;
+export const MAX_DIFF_TOKENS = 0x40000;
+
+// diff-match-patch's JavaScript port operates on UTF-16 code units, not
+// Unicode code points. Keep its private encoding in the BMP while skipping
+// surrogates; larger token sets take the deterministic token-array fallback.
+const BMP_FIRST_CODE = 1;
+const BMP_BEFORE_SURROGATES = 0xD800 - BMP_FIRST_CODE;
+const BMP_AFTER_SURROGATES = 0x10000 - 0xE000;
+const DMP_SAFE_TOKEN_LIMIT = BMP_BEFORE_SURROGATES + BMP_AFTER_SURROGATES;
+
+export class DiffTokenLimitError extends Error {
+    constructor(limit = MAX_DIFF_TOKENS) {
+        super(`Word diff exceeds the safe limit of ${limit} unique tokens.`);
+        this.name = 'DiffTokenLimitError';
+        this.code = 'DIFF_TOKEN_LIMIT';
+        this.limit = limit;
+    }
+}
+
+export function isDiffTokenLimitError(error) {
+    return error?.code === 'DIFF_TOKEN_LIMIT';
+}
+
+export function createDiffEngine(options = {}) {
+    const timeout = options.diffTimeoutSeconds ?? 0;
+    if (!Number.isFinite(timeout) || timeout < 0) {
+        throw new TypeError('diffTimeoutSeconds must be a finite non-negative number.');
+    }
+    const engine = new diff_match_patch();
+    engine.Diff_Timeout = timeout;
+    return engine;
+}
+
+function tokenize(text, atomicChars = null) {
+    const tokens = [];
+    const pushWord = word => {
+        if (!atomicChars || atomicChars.size === 0) {
+            tokens.push(word);
+            return;
+        }
+        // Structural placeholders (for example manual line breaks) stand
+        // alone so a change beside one never swallows text across it.
+        let start = 0;
+        for (let index = 0; index < word.length; index++) {
+            if (!atomicChars.has(word[index])) continue;
+            if (index > start) tokens.push(word.slice(start, index));
+            tokens.push(word[index]);
+            start = index + 1;
+        }
+        if (start < word.length) tokens.push(word.slice(start));
+    };
+    const leading = text.match(/^\s+/);
+    if (leading) tokens.push(leading[0]);
+
+    const regex = /(\S+)(\s*)/g;
+    regex.lastIndex = leading?.[0].length || 0;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        if (match[1]) pushWord(match[1]);
+        if (match[2]) tokens.push(match[2]);
+    }
+    return tokens;
+}
+
 /**
  * Converts text into word tokens represented as unique characters.
  * This allows DMP to diff at word-level instead of character-level.
  * 
  * @param {string} text1 - First text to tokenize
  * @param {string} text2 - Second text to tokenize
- * @param {{ maxTokens?: number }} [options={}] - Internal/test capacity override
- * @returns {{ chars1: string, chars2: string, wordArray: string[], tokenIds1: number[], tokenIds2: number[] }}
- */
-export function wordsToChars(text1, text2, options = {}) {
-    const wordArray = [];
-    const wordHash = new Map();
-    const maxTokens = options.maxTokens ?? MAX_DIFF_TOKENS;
-    if (!Number.isInteger(maxTokens) || maxTokens < 1 || maxTokens > MAX_DIFF_TOKENS) {
-        throw new RangeError(`maxTokens must be an integer from 1 to ${MAX_DIFF_TOKENS}.`);
-    }
-
-    function mapTokensToChars(tokens) {
-        let chars = '';
-        const tokenIds = [];
-        for (const token of tokens) {
-            let tokenId = wordHash.get(token);
-            if (tokenId === undefined) {
-                if (wordArray.length >= maxTokens) throw new DiffTokenLimitError(maxTokens);
-                tokenId = wordArray.length;
-                wordArray.push(token);
-                wordHash.set(token, tokenId);
-            }
-            tokenIds.push(tokenId);
-            chars += String.fromCodePoint(TOKEN_CODE_POINT_BASE + tokenId);
-        }
-        return { chars, tokenIds };
-    }
-
-    const tokens1 = tokenize(text1);
-    const tokens2 = tokenize(text2);
-    const encoded1 = mapTokensToChars(tokens1);
-    const encoded2 = mapTokensToChars(tokens2);
-
-    return {
-        chars1: encoded1.chars,
-        chars2: encoded2.chars,
-        wordArray,
-        tokenIds1: encoded1.tokenIds,
-        tokenIds2: encoded2.tokenIds
-    };
-}
+ * @param {{ maxTokens?: number, atomicChars?: Set<string> }} [options={}] - Capacity override and single-character tokens
+ * @returns {{ chars1: string, chars2: string, wordArray: string[], tokenIds1: number[], tokenIds2: number[] }}
+ */
+export function wordsToChars(text1, text2, options = {}) {
+    const wordArray = [];
+    const wordHash = new Map();
+    const maxTokens = options.maxTokens ?? MAX_DIFF_TOKENS;
+    if (!Number.isInteger(maxTokens) || maxTokens < 1 || maxTokens > MAX_DIFF_TOKENS) {
+        throw new RangeError(`maxTokens must be an integer from 1 to ${MAX_DIFF_TOKENS}.`);
+    }
+
+    function mapTokensToChars(tokens) {
+        let chars = '';
+        const tokenIds = [];
+        for (const token of tokens) {
+            let tokenId = wordHash.get(token);
+            if (tokenId === undefined) {
+                if (wordArray.length >= maxTokens) throw new DiffTokenLimitError(maxTokens);
+                tokenId = wordArray.length;
+                wordArray.push(token);
+                wordHash.set(token, tokenId);
+            }
+            tokenIds.push(tokenId);
+            chars += String.fromCodePoint(TOKEN_CODE_POINT_BASE + tokenId);
+        }
+        return { chars, tokenIds };
+    }
+
+    const tokens1 = tokenize(text1, options.atomicChars);
+    const tokens2 = tokenize(text2, options.atomicChars);
+    const encoded1 = mapTokensToChars(tokens1);
+    const encoded2 = mapTokensToChars(tokens2);
+
+    return {
+        chars1: encoded1.chars,
+        chars2: encoded2.chars,
+        wordArray,
+        tokenIds1: encoded1.tokenIds,
+        tokenIds2: encoded2.tokenIds
+    };
+}
 
 /**
  * Converts character-encoded diffs back to actual word diffs.
@@ -111,173 +127,173 @@ export function wordsToChars(text1, text2, options = {}) {
  * @param {string[]} wordArray - Array mapping char codes to words
  * @returns {Array} DMP-style diff array with actual words
  */
-export function charsToWords(diffs, wordArray) {
-    const originalChars = diffs.filter(([op]) => op !== 1).map(([, chars]) => chars).join('');
-    const modifiedChars = diffs.filter(([op]) => op !== -1).map(([, chars]) => chars).join('');
-
-    const decode = chars => {
-        const tokenIds = [];
-        for (let i = 0; i < chars.length;) {
-            const codePoint = chars.codePointAt(i);
-            const tokenId = codePoint - TOKEN_CODE_POINT_BASE;
-            if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= wordArray.length) {
-                throw new RangeError(`Diff token code point U+${codePoint.toString(16).toUpperCase()} has no mapping.`);
-            }
-            tokenIds.push(tokenId);
-            i += codePoint > 0xFFFF ? 2 : 1;
-        }
-        return tokenIds;
-    };
-
-    // Reconstruct both encoded sides before decoding. A UTF-16-based diff
-    // engine may split a surrogate pair across adjacent diff tuples even
-    // though the complete original/modified streams remain valid.
-    return deterministicLargeTokenDiff(decode(originalChars), decode(modifiedChars), wordArray);
-}
-
-function tokenIdToBmpChar(tokenId) {
-    const code = tokenId < BMP_BEFORE_SURROGATES
-        ? BMP_FIRST_CODE + tokenId
-        : 0xE000 + (tokenId - BMP_BEFORE_SURROGATES);
-    return String.fromCharCode(code);
-}
-
-function bmpCharToTokenId(charCode) {
-    if (charCode >= BMP_FIRST_CODE && charCode < 0xD800) return charCode - BMP_FIRST_CODE;
-    if (charCode >= 0xE000 && charCode <= 0xFFFF) return BMP_BEFORE_SURROGATES + charCode - 0xE000;
-    throw new RangeError(`BMP diff token U+${charCode.toString(16).toUpperCase()} has no mapping.`);
-}
-
-function encodeTokenIdsForDmp(tokenIds) {
-    let chars = '';
-    for (const tokenId of tokenIds) chars += tokenIdToBmpChar(tokenId);
-    return chars;
-}
-
-function decodeBmpDiffs(diffs, wordArray) {
-    return diffs.map(([op, chars]) => {
-        const parts = [];
-        for (let index = 0; index < chars.length; index++) {
-            const tokenId = bmpCharToTokenId(chars.charCodeAt(index));
-            if (tokenId >= wordArray.length) {
-                throw new RangeError(`BMP diff token ${tokenId} has no mapping.`);
-            }
-            parts.push(wordArray[tokenId]);
-        }
-        return [op, parts.join('')];
-    });
-}
-
-function deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray) {
-    let prefixLength = 0;
-    const sharedLength = Math.min(tokenIds1.length, tokenIds2.length);
-    while (prefixLength < sharedLength && tokenIds1[prefixLength] === tokenIds2[prefixLength]) {
-        prefixLength++;
-    }
-
-    let suffixLength = 0;
-    while (
-        suffixLength < sharedLength - prefixLength
-        && tokenIds1[tokenIds1.length - 1 - suffixLength] === tokenIds2[tokenIds2.length - 1 - suffixLength]
-    ) {
-        suffixLength++;
-    }
-
-    const joinTokens = ids => ids.map(id => wordArray[id]).join('');
-    const diffs = [];
-    if (prefixLength) diffs.push([0, joinTokens(tokenIds1.slice(0, prefixLength))]);
-
-    const deleted = tokenIds1.slice(prefixLength, tokenIds1.length - suffixLength);
-    const inserted = tokenIds2.slice(prefixLength, tokenIds2.length - suffixLength);
-    if (deleted.length) diffs.push([-1, joinTokens(deleted)]);
-    if (inserted.length) diffs.push([1, joinTokens(inserted)]);
-    if (suffixLength) diffs.push([0, joinTokens(tokenIds1.slice(tokenIds1.length - suffixLength))]);
-    return diffs;
-}
-
-/**
- * Computes word-level diff tuples using a shared diff engine instance.
- *
- * @param {string} originalText - Original text
- * @param {string} newText - New text
- * @param {{ cleanupSemantic?: boolean, diffTimeoutSeconds?: number, maxTokens?: number }} [options={}] - Diff options
- * @returns {Array<[number, string]>}
- */
-export function computeWordDiffs(originalText, newText, options = {}) {
-    if (originalText === newText) {
-        return [[0, originalText]];
-    }
-
-    if (!originalText) {
-        return [[1, newText]];
-    }
-
-    if (!newText) {
-        return [[-1, originalText]];
-    }
-
-    const { cleanupSemantic = true } = options;
-
-    const { wordArray, tokenIds1, tokenIds2 } = wordsToChars(originalText, newText, options);
-    if (wordArray.length > DMP_SAFE_TOKEN_LIMIT) {
-        return deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray);
-    }
-
-    const dmp = createDiffEngine(options);
-    const charDiffs = dmp.diff_main(encodeTokenIdsForDmp(tokenIds1), encodeTokenIdsForDmp(tokenIds2));
-    if (cleanupSemantic) {
-        dmp.diff_cleanupSemantic(charDiffs);
-    }
-
-    return decodeBmpDiffs(charDiffs, wordArray);
-}
-
-/**
- * Computes a character-local diff without semantic cleanup. This is used to
- * refine whitespace-only substitutions that a word-level token groups with
- * adjacent unchanged content (for example, an NBSP beside a hyperlink).
- *
- * @param {string} originalText
- * @param {string} newText
- * @param {{ diffTimeoutSeconds?: number }} [options={}]
- * @returns {Array<[number, string]>}
- */
-export function computeCharacterDiffs(originalText, newText, options = {}) {
-    if (originalText === newText) return [[0, originalText]];
-    return createDiffEngine(options).diff_main(originalText, newText);
-}
-
-/**
- * Returns a character-local diff only when the modified string can be made
- * solely by inserting into the original. This prevents word-token cleanup
- * from relocating small insertions between repeated phrases or containers.
- *
- * @param {string} originalText
- * @param {string} newText
- * @returns {Array<[number, string]>|null}
- */
-export function computeInsertionOnlyDiffs(originalText, newText) {
-    if (originalText === newText) return [[0, originalText]];
-    if (!originalText) return [[1, newText]];
-    let originalIndex = 0;
-    for (let modifiedIndex = 0; modifiedIndex < newText.length && originalIndex < originalText.length; modifiedIndex++) {
-        if (newText[modifiedIndex] === originalText[originalIndex]) originalIndex++;
-    }
-    if (originalIndex !== originalText.length) return null;
-
-    const diffs = createDiffEngine().diff_main(originalText, newText);
-    return diffs.some(([op]) => op === -1) ? null : diffs;
-}
-
-/**
- * Computes word-level diff operations with offset tracking.
+export function charsToWords(diffs, wordArray) {
+    const originalChars = diffs.filter(([op]) => op !== 1).map(([, chars]) => chars).join('');
+    const modifiedChars = diffs.filter(([op]) => op !== -1).map(([, chars]) => chars).join('');
+
+    const decode = chars => {
+        const tokenIds = [];
+        for (let i = 0; i < chars.length;) {
+            const codePoint = chars.codePointAt(i);
+            const tokenId = codePoint - TOKEN_CODE_POINT_BASE;
+            if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= wordArray.length) {
+                throw new RangeError(`Diff token code point U+${codePoint.toString(16).toUpperCase()} has no mapping.`);
+            }
+            tokenIds.push(tokenId);
+            i += codePoint > 0xFFFF ? 2 : 1;
+        }
+        return tokenIds;
+    };
+
+    // Reconstruct both encoded sides before decoding. A UTF-16-based diff
+    // engine may split a surrogate pair across adjacent diff tuples even
+    // though the complete original/modified streams remain valid.
+    return deterministicLargeTokenDiff(decode(originalChars), decode(modifiedChars), wordArray);
+}
+
+function tokenIdToBmpChar(tokenId) {
+    const code = tokenId < BMP_BEFORE_SURROGATES
+        ? BMP_FIRST_CODE + tokenId
+        : 0xE000 + (tokenId - BMP_BEFORE_SURROGATES);
+    return String.fromCharCode(code);
+}
+
+function bmpCharToTokenId(charCode) {
+    if (charCode >= BMP_FIRST_CODE && charCode < 0xD800) return charCode - BMP_FIRST_CODE;
+    if (charCode >= 0xE000 && charCode <= 0xFFFF) return BMP_BEFORE_SURROGATES + charCode - 0xE000;
+    throw new RangeError(`BMP diff token U+${charCode.toString(16).toUpperCase()} has no mapping.`);
+}
+
+function encodeTokenIdsForDmp(tokenIds) {
+    let chars = '';
+    for (const tokenId of tokenIds) chars += tokenIdToBmpChar(tokenId);
+    return chars;
+}
+
+function decodeBmpDiffs(diffs, wordArray) {
+    return diffs.map(([op, chars]) => {
+        const parts = [];
+        for (let index = 0; index < chars.length; index++) {
+            const tokenId = bmpCharToTokenId(chars.charCodeAt(index));
+            if (tokenId >= wordArray.length) {
+                throw new RangeError(`BMP diff token ${tokenId} has no mapping.`);
+            }
+            parts.push(wordArray[tokenId]);
+        }
+        return [op, parts.join('')];
+    });
+}
+
+function deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray) {
+    let prefixLength = 0;
+    const sharedLength = Math.min(tokenIds1.length, tokenIds2.length);
+    while (prefixLength < sharedLength && tokenIds1[prefixLength] === tokenIds2[prefixLength]) {
+        prefixLength++;
+    }
+
+    let suffixLength = 0;
+    while (
+        suffixLength < sharedLength - prefixLength
+        && tokenIds1[tokenIds1.length - 1 - suffixLength] === tokenIds2[tokenIds2.length - 1 - suffixLength]
+    ) {
+        suffixLength++;
+    }
+
+    const joinTokens = ids => ids.map(id => wordArray[id]).join('');
+    const diffs = [];
+    if (prefixLength) diffs.push([0, joinTokens(tokenIds1.slice(0, prefixLength))]);
+
+    const deleted = tokenIds1.slice(prefixLength, tokenIds1.length - suffixLength);
+    const inserted = tokenIds2.slice(prefixLength, tokenIds2.length - suffixLength);
+    if (deleted.length) diffs.push([-1, joinTokens(deleted)]);
+    if (inserted.length) diffs.push([1, joinTokens(inserted)]);
+    if (suffixLength) diffs.push([0, joinTokens(tokenIds1.slice(tokenIds1.length - suffixLength))]);
+    return diffs;
+}
+
+/**
+ * Computes word-level diff tuples using a shared diff engine instance.
+ *
+ * @param {string} originalText - Original text
+ * @param {string} newText - New text
+ * @param {{ cleanupSemantic?: boolean, diffTimeoutSeconds?: number, maxTokens?: number, atomicChars?: Set<string> }} [options={}] - Diff options
+ * @returns {Array<[number, string]>}
+ */
+export function computeWordDiffs(originalText, newText, options = {}) {
+    if (originalText === newText) {
+        return [[0, originalText]];
+    }
+
+    if (!originalText) {
+        return [[1, newText]];
+    }
+
+    if (!newText) {
+        return [[-1, originalText]];
+    }
+
+    const { cleanupSemantic = true } = options;
+
+    const { wordArray, tokenIds1, tokenIds2 } = wordsToChars(originalText, newText, options);
+    if (wordArray.length > DMP_SAFE_TOKEN_LIMIT) {
+        return deterministicLargeTokenDiff(tokenIds1, tokenIds2, wordArray);
+    }
+
+    const dmp = createDiffEngine(options);
+    const charDiffs = dmp.diff_main(encodeTokenIdsForDmp(tokenIds1), encodeTokenIdsForDmp(tokenIds2));
+    if (cleanupSemantic) {
+        dmp.diff_cleanupSemantic(charDiffs);
+    }
+
+    return decodeBmpDiffs(charDiffs, wordArray);
+}
+
+/**
+ * Computes a character-local diff without semantic cleanup. This is used to
+ * refine whitespace-only substitutions that a word-level token groups with
+ * adjacent unchanged content (for example, an NBSP beside a hyperlink).
+ *
+ * @param {string} originalText
+ * @param {string} newText
+ * @param {{ diffTimeoutSeconds?: number }} [options={}]
+ * @returns {Array<[number, string]>}
+ */
+export function computeCharacterDiffs(originalText, newText, options = {}) {
+    if (originalText === newText) return [[0, originalText]];
+    return createDiffEngine(options).diff_main(originalText, newText);
+}
+
+/**
+ * Returns a character-local diff only when the modified string can be made
+ * solely by inserting into the original. This prevents word-token cleanup
+ * from relocating small insertions between repeated phrases or containers.
+ *
+ * @param {string} originalText
+ * @param {string} newText
+ * @returns {Array<[number, string]>|null}
+ */
+export function computeInsertionOnlyDiffs(originalText, newText) {
+    if (originalText === newText) return [[0, originalText]];
+    if (!originalText) return [[1, newText]];
+    let originalIndex = 0;
+    for (let modifiedIndex = 0; modifiedIndex < newText.length && originalIndex < originalText.length; modifiedIndex++) {
+        if (newText[modifiedIndex] === originalText[originalIndex]) originalIndex++;
+    }
+    if (originalIndex !== originalText.length) return null;
+
+    const diffs = createDiffEngine().diff_main(originalText, newText);
+    return diffs.some(([op]) => op === -1) ? null : diffs;
+}
+
+/**
+ * Computes word-level diff operations with offset tracking.
  * 
  * @param {string} originalText - Original text
  * @param {string} newText - New text
- * @param {{ cleanupSemantic?: boolean }} [options={}] - Diff options
- * @returns {import('../core/types.js').DiffOperation[]}
- */
-export function computeWordLevelDiffOps(originalText, newText, options = {}) {
+ * @param {{ cleanupSemantic?: boolean }} [options={}] - Diff options
+ * @returns {import('../core/types.js').DiffOperation[]}
+ */
+export function computeWordLevelDiffOps(originalText, newText, options = {}) {
     // Handle edge cases
     if (originalText === newText) {
         return [{
@@ -306,7 +322,7 @@ export function computeWordLevelDiffOps(originalText, newText, options = {}) {
         }];
     }
 
-    const wordDiffs = computeWordDiffs(originalText, newText, options);
+    const wordDiffs = computeWordDiffs(originalText, newText, options);
 
     // Convert to operations with offsets
     const operations = [];