@stll/folio-core 0.47.6 → 0.48.0

package/dist/ai-edits/word-diff.d.ts

@@ -2,8 +2,9 @@
 /**
  * Diff between two strings, as the segments a redline is drawn from.
  *
- * Tokenises (by default on whitespace boundaries, preserving the whitespace as
- * part of each token), runs an LCS, and returns a left-to-right ordered list of
+ * Tokenises (by default into words and the punctuation marks at their edges,
+ * preserving the whitespace as part of each token; see {@link pushRunTokens}),
+ * runs an LCS, and returns a left-to-right ordered list of
  * segments where shared runs render as `equal`, removed runs as `del`, and
  * added runs as `ins`. Used by the panel (to render minimal-change redlines),
  * the version comparison, and the apply engine (so tracked changes mark only
@@ -18,10 +19,25 @@
  * same thing and can actually be read. Three rules pull the output back:
  *
  * 1. A match made only of separators is not a match ({@link isSeparatorOnly}).
- * 2. A match too short to carry meaning is dropped unless it opens the string,
- *    where it is the reader's anchor rather than an island.
+ * 2. A match too short to carry meaning is dropped when changes sit on both
+ *    sides of it and at least one of them changes words. At either end of the
+ *    string it is the reader's anchor, and
+ *    between two punctuation or whitespace edits it is what they were made
+ *    around; neither is an island.
  * 3. When what survives is still too fragmented for its length, the whole
- *    paragraph is one replacement ({@link isTooFragmented}).
+ *    paragraph is one replacement ({@link isTooFragmented}). Punctuation and
+ *    whitespace edits alone never are.
+ *
+ * At word granularity the LCS itself prefers words: one matched word outweighs
+ * any number of matched marks, and a unique mark is never an anchor.
+ *
+ * A word token carries the whitespace before it, so a string's first word has
+ * none while the same word inside the other string does. Word tokens are
+ * therefore matched on their text alone; the rules above judge that
+ * alignment, and a whitespace difference around a matched word is marked
+ * afterwards as a change of the whitespace only
+ * ({@link separateWhitespaceChanges}). Otherwise prefixing "(1) " would strike
+ * through the unchanged first word.
  *
  * Common affixes and unique-token anchors split the input into independent
  * gaps before any quadratic work. The residual gaps share one
@@ -61,6 +77,9 @@ type WordDiffOptions = {
   /** Default: nothing normalized, so both strings reconstruct exactly. */
   normalization?: WordDiffNormalization;
 };
+/** Keep token storage bounded when a run has an unusually long punctuation edge. */
+declare const MAX_EDGE_PUNCTUATION_TOKENS = 64;
+declare const tokenizeWords: (value: string) => string[];
 /**
  * One internal comparison/apply scope. Every diff shares the same quadratic
  * allowance; public standalone calls below still receive a fresh allowance.
@@ -75,4 +94,4 @@ declare const createScopedWordDiffOptions: <Options extends WordDiffOptions>(opt
 declare const wordDiffSessionFromOptions: (options: WordDiffOptions | undefined) => ReturnType<typeof createWordDiffSession>;
 declare const diffWordSegments: (before: string, after: string, options?: WordDiffOptions) => WordDiffSegment[];
 //#endregion
-export { WORD_DIFF_GRANULARITIES, WordDiffGranularity, WordDiffNormalization, WordDiffOptions, WordDiffSegment, createScopedWordDiffOptions, createWordDiffSession, diffWordSegments, wordDiffSessionFromOptions };
+export { MAX_EDGE_PUNCTUATION_TOKENS, WORD_DIFF_GRANULARITIES, WordDiffGranularity, WordDiffNormalization, WordDiffOptions, WordDiffSegment, createScopedWordDiffOptions, createWordDiffSession, diffWordSegments, tokenizeWords, wordDiffSessionFromOptions };

package/dist/ai-edits/word-diff.js

@@ -7,10 +7,11 @@
 */
 const WORD_DIFF_GRANULARITIES = Object.freeze(["word", "character"]);
 const WHITESPACE = /\s/u;
-/** Punctuation, symbols and whitespace: everything that is not content. */
-const SEPARATOR_ONLY = /^[\s\p{P}\p{S}]*$/u;
+/** Punctuation and whitespace carry no content except legal section/paragraph signs. */
+const SEPARATOR_ONLY = /^[\s\p{P}]*$/u;
+const CONTENT_PUNCTUATION = /[§¶]/u;
 /**
-* A match of at most this many tokens that carries no letters or digits is
+* A match of at most this many tokens that carries only punctuation or space is
 * noise: a lone space, a comma, a stray closing bracket. Matching it splits
 * two rewrites into four.
 */
@@ -33,6 +34,59 @@ const MINIMUM_ISOLATED_MATCH_UNITS = 2;
 const FRAGMENTATION_SCALE = 32;
 /** Below this length every match is a large fraction of the string, so the floor says nothing. */
 const FRAGMENTATION_MINIMUM_AVERAGE_LENGTH = 8;
+/** Unicode general category P: punctuation marks, dashes, brackets and quotes. */
+const PUNCTUATION = /^\p{P}$/u;
+/** Keep token storage bounded when a run has an unusually long punctuation edge. */
+const MAX_EDGE_PUNCTUATION_TOKENS = 64;
+/** The code point starting at `index`, as a string of one or two UTF-16 units. */
+const codePointAt = (value, index) => String.fromCodePoint(value.codePointAt(index) ?? 0);
+/** The code point ending just before `end`. */
+const codePointBefore = (value, end) => {
+	const low = value.charCodeAt(end - 1);
+	const high = value.charCodeAt(end - 2);
+	return low >= 56320 && low <= 57343 && high >= 55296 && high <= 56319 ? value.slice(end - 2, end) : value.slice(end - 1, end);
+};
+/**
+* Split one whitespace-free run into its words and punctuation marks.
+*
+* Punctuation marks (general category P) at either edge of the run are
+* tokens of their own, as Word's compare treats them: `jmění.` becoming `jmění,`
+* changes the mark, not the word. Punctuation inside the run stays in the
+* word, so `d.o.o`, `1.1.2026`, `3.5`, `well-known` and `don't` are single
+* tokens; only their edge marks split off (`d.o.o.` is `d.o.o` + `.`, `b)` is
+* `b` + `)`). A run made only of punctuation is split up to the edge cap.
+* `§` and `¶` are content markers despite their Unicode punctuation category;
+* symbols (general category S, such as `€` or `+`) are not punctuation.
+*/
+const pushRunTokens = (tokens, value, run, prefixStart) => {
+	let wordStart = run.start;
+	let pending = value.slice(prefixStart, run.start);
+	let leadingMarks = 0;
+	while (wordStart < run.end && leadingMarks < 64) {
+		const mark = codePointAt(value, wordStart);
+		if (!PUNCTUATION.test(mark) || CONTENT_PUNCTUATION.test(mark)) break;
+		tokens.push(pending + mark);
+		pending = "";
+		wordStart += mark.length;
+		leadingMarks++;
+	}
+	let wordEnd = run.end;
+	const trailing = [];
+	while (wordEnd > wordStart && trailing.length < 64) {
+		const mark = codePointBefore(value, wordEnd);
+		if (!PUNCTUATION.test(mark) || CONTENT_PUNCTUATION.test(mark)) break;
+		trailing.push(mark);
+		wordEnd -= mark.length;
+	}
+	if (wordStart < wordEnd) {
+		tokens.push(pending + value.slice(wordStart, wordEnd));
+		pending = "";
+	}
+	for (let index = trailing.length - 1; index >= 0; index--) {
+		tokens.push(pending + (trailing[index] ?? ""));
+		pending = "";
+	}
+};
 const tokenizeWords = (value) => {
 	const tokens = [];
 	let tokenStart = 0;
@@ -40,8 +94,12 @@ const tokenizeWords = (value) => {
 	while (cursor < value.length) {
 		while (cursor < value.length && WHITESPACE.test(value.charAt(cursor))) cursor++;
 		if (cursor === value.length) break;
+		const runStart = cursor;
 		while (cursor < value.length && !WHITESPACE.test(value.charAt(cursor))) cursor++;
-		tokens.push(value.slice(tokenStart, cursor));
+		pushRunTokens(tokens, value, {
+			start: runStart,
+			end: cursor
+		}, tokenStart);
 		tokenStart = cursor;
 	}
 	const last = tokens.at(-1);
@@ -61,7 +119,7 @@ const comparisonKey = (token, normalization) => {
 	const collapsed = normalization.whitespace === true ? token.replaceAll(/\s+/gu, " ").trim() : token;
 	return normalization.case === true ? collapsed.toLowerCase() : collapsed;
 };
-const isSeparatorOnly = (text) => SEPARATOR_ONLY.test(text);
+const isSeparatorOnly = (text) => SEPARATOR_ONLY.test(text) && !CONTENT_PUNCTUATION.test(text);
 /**
 * Cell budget shared by the residual LCS gaps inside one comparison or apply
 * scope. `before`/`after` come from attacker-controlled document text (a
@@ -85,11 +143,21 @@ const MAX_WORD_DIFF_ANCHOR_TOKENS = 16384;
 * and case or whitespace normalization creates another string for it.
 */
 const MAX_WORD_DIFF_COMPARISON_KEY_CODE_UNITS = 1048576;
+/**
+* How the LCS scores a match. `uniform` counts every matched token once;
+* `content-first` (word granularity) never trades a matched word for matched
+* punctuation, so moving a mark across a word cannot strike the word through.
+*/
+const MATCH_WEIGHTING = {
+	Uniform: "uniform",
+	ContentFirst: "content-first"
+};
 const ALIGNMENT_OPERATION = {
 	Equal: 1,
 	Delete: 2,
 	Insert: 3
 };
+const hasOppositeChange = (runs, direction) => runs.some(({ type }) => type === (direction === "insertion" ? "del" : "ins"));
 const fitsCellBudget = (beforeLength, afterLength, budget) => beforeLength === 0 || afterLength === 0 || beforeLength <= Math.floor(budget.remainingCells / afterLength);
 const pushRun = (runs, run) => {
 	const last = runs.at(-1);
@@ -128,13 +196,7 @@ const pushChangedRange = (runs, before, after, beforeRange, afterRange) => {
 		text: joinTokenRange(after, afterRange)
 	});
 };
-/**
-* Unique tokens common to both ranges, reduced to a monotone subsequence of
-* target positions. Repeated boilerplate is deliberately ineligible: a unique
-* clause number or name is stronger lineage evidence than another occurrence
-* of "the" chosen by an arbitrary LCS tie.
-*/
-const findPatienceAnchors = (beforeKeys, afterKeys, beforeRange, afterRange) => {
+const findPatienceAnchors = ({ beforeKeys, afterKeys, beforeRange, afterRange, weighting }) => {
 	const beforeOccurrences = /* @__PURE__ */ new Map();
 	for (let index = beforeRange.start; index < beforeRange.end; index++) {
 		const key = beforeKeys[index] ?? "";
@@ -155,7 +217,7 @@ const findPatienceAnchors = (beforeKeys, afterKeys, beforeRange, afterRange) =>
 		const key = beforeKeys[beforeIndex] ?? "";
 		const beforeCount = beforeOccurrences.get(key);
 		const afterOccurrence = afterOccurrences.get(key);
-		if (beforeCount === 1 && afterOccurrence?.count === 1) candidates.push({
+		if ((weighting === MATCH_WEIGHTING.Uniform || !isSeparatorOnly(key)) && beforeCount === 1 && afterOccurrence?.count === 1) candidates.push({
 			beforeIndex,
 			afterIndex: afterOccurrence.index
 		});
@@ -275,7 +337,7 @@ const alignMonotoneChange = (beforeText, afterText, before, after, normalization
 	return runs;
 };
 /** Longest-common-subsequence alignment for one residual, bounded gap. */
-const alignDenseGap = ({ before, after, beforeKeys, afterKeys, beforeRange, afterRange, runs, budget }) => {
+const alignDenseGap = ({ before, after, beforeKeys, afterKeys, beforeRange, afterRange, runs, budget, weighting }) => {
 	const beforeLength = beforeRange.end - beforeRange.start;
 	const afterLength = afterRange.end - afterRange.start;
 	if (beforeLength === 0 || afterLength === 0) {
@@ -289,15 +351,24 @@ const alignDenseGap = ({ before, after, beforeKeys, afterKeys, beforeRange, afte
 	const cellCount = beforeLength * afterLength;
 	budget.remainingCells -= cellCount;
 	const lengths = new Uint32Array(cellCount);
-	for (let beforeOffset = 0; beforeOffset < beforeLength; beforeOffset++) for (let afterOffset = 0; afterOffset < afterLength; afterOffset++) {
-		const index = beforeOffset * afterLength + afterOffset;
-		if (beforeKeys[beforeRange.start + beforeOffset] === afterKeys[afterRange.start + afterOffset]) {
-			lengths[index] = (beforeOffset > 0 && afterOffset > 0 ? lengths[(beforeOffset - 1) * afterLength + afterOffset - 1] ?? 0 : 0) + 1;
-			continue;
+	const contentWeight = weighting === MATCH_WEIGHTING.ContentFirst ? Math.min(beforeLength, afterLength) + 1 : 1;
+	const matchWeights = new Uint32Array(beforeLength);
+	for (let beforeOffset = 0; beforeOffset < beforeLength; beforeOffset++) {
+		const key = beforeKeys[beforeRange.start + beforeOffset] ?? "";
+		matchWeights[beforeOffset] = isSeparatorOnly(key) ? 1 : contentWeight;
+	}
+	for (let beforeOffset = 0; beforeOffset < beforeLength; beforeOffset++) {
+		const matchWeight = matchWeights[beforeOffset] ?? 1;
+		for (let afterOffset = 0; afterOffset < afterLength; afterOffset++) {
+			const index = beforeOffset * afterLength + afterOffset;
+			if (beforeKeys[beforeRange.start + beforeOffset] === afterKeys[afterRange.start + afterOffset]) {
+				lengths[index] = (beforeOffset > 0 && afterOffset > 0 ? lengths[(beforeOffset - 1) * afterLength + afterOffset - 1] ?? 0 : 0) + matchWeight;
+				continue;
+			}
+			const above = beforeOffset > 0 ? lengths[(beforeOffset - 1) * afterLength + afterOffset] ?? 0 : 0;
+			const left = afterOffset > 0 ? lengths[index - 1] ?? 0 : 0;
+			lengths[index] = Math.max(above, left);
 		}
-		const above = beforeOffset > 0 ? lengths[(beforeOffset - 1) * afterLength + afterOffset] ?? 0 : 0;
-		const left = afterOffset > 0 ? lengths[index - 1] ?? 0 : 0;
-		lengths[index] = Math.max(above, left);
 	}
 	const reversedOperations = new Uint8Array(beforeLength + afterLength);
 	let operationCount = 0;
@@ -404,7 +475,7 @@ const alignGap = (options) => {
 * repeated boilerplate from winning a tie over a unique legal term, and makes
 * a small edit inside a long paragraph pay for only its changed gap.
 */
-const alignTokens = ({ beforeText, afterText, before, after, normalization, budget }) => {
+const alignTokens = ({ beforeText, afterText, before, after, normalization, budget, weighting }) => {
 	let commonPrefixLength = 0;
 	let beforePrefixLength = 0;
 	let afterPrefixLength = 0;
@@ -491,7 +562,13 @@ const alignTokens = ({ beforeText, afterText, before, after, normalization, budg
 		start: 0,
 		end: afterMiddle.length
 	};
-	const anchors = findPatienceAnchors(beforeKeys, afterKeys, middleRangeBefore, middleRangeAfter);
+	const anchors = findPatienceAnchors({
+		beforeKeys,
+		afterKeys,
+		beforeRange: middleRangeBefore,
+		afterRange: middleRangeAfter,
+		weighting
+	});
 	if (anchors.length === 0 && fullInputFits && fullInputFitsStorage) {
 		alignDenseGap({
 			before,
@@ -507,7 +584,8 @@ const alignTokens = ({ beforeText, afterText, before, after, normalization, budg
 				end: after.length
 			},
 			runs,
-			budget
+			budget,
+			weighting
 		});
 		return {
 			runs,
@@ -538,7 +616,8 @@ const alignTokens = ({ beforeText, afterText, before, after, normalization, budg
 				end: anchor.afterIndex
 			},
 			runs,
-			budget
+			budget,
+			weighting
 		});
 		pushRun(runs, {
 			type: "equal",
@@ -563,7 +642,8 @@ const alignTokens = ({ beforeText, afterText, before, after, normalization, budg
 			end: afterMiddle.length
 		},
 		runs,
-		budget
+		budget,
+		weighting
 	});
 	pushEqualRange(runs, before, after, {
 		start: beforeMiddleEnd,
@@ -587,6 +667,16 @@ const isTooFragmented = (runs, averageLength) => {
 	for (const run of runs) if (run.type === "equal") sumOfSquares += run.before.length * run.before.length;
 	return sumOfSquares * FRAGMENTATION_SCALE < averageLength * averageLength;
 };
+/** True when a run changes words, not only punctuation or whitespace. */
+const isContentChange = (run) => run.type !== "equal" && !isSeparatorOnly(run.text);
+/** True when the change region starting at `from`, walking by `step`, changes words. */
+const changeCarriesContent = (runs, { from, step }) => {
+	for (let index = from;; index += step) {
+		const run = runs[index];
+		if (run === void 0 || run.type === "equal") return false;
+		if (isContentChange(run)) return true;
+	}
+};
 /**
 * Turn a match the quality rules rejected back into the change it interrupts.
 *
@@ -602,7 +692,13 @@ const demoteRejectedMatches = (runs) => {
 			continue;
 		}
 		const interruptsAChange = runs[index - 1] !== void 0 || runs[index + 1] !== void 0;
-		const isIsland = runs[index - 1] !== void 0 && runs[index + 1] !== void 0;
+		const isIsland = runs[index - 1] !== void 0 && runs[index + 1] !== void 0 && (changeCarriesContent(runs, {
+			from: index - 1,
+			step: -1
+		}) || changeCarriesContent(runs, {
+			from: index + 1,
+			step: 1
+		}));
 		if (!(interruptsAChange && run.units <= MAX_SEPARATOR_ONLY_MATCH_UNITS && isSeparatorOnly(run.before) || isIsland && run.units < MINIMUM_ISOLATED_MATCH_UNITS)) {
 			kept.push(run);
 			continue;
@@ -617,6 +713,70 @@ const demoteRejectedMatches = (runs) => {
 	}
 	return kept;
 };
+const TOKEN_WHITESPACE = /^(\s*)(.*?)(\s*)$/su;
+const splitTokenWhitespace = (token) => {
+	const [, leading = "", text = "", trailing = ""] = TOKEN_WHITESPACE.exec(token) ?? [];
+	return {
+		leading,
+		text,
+		trailing
+	};
+};
+const pushChangedText = (runs, before, after) => {
+	if (before.length > 0) pushRun(runs, {
+		type: "del",
+		text: before
+	});
+	if (after.length > 0) pushRun(runs, {
+		type: "ins",
+		text: after
+	});
+};
+const pushWhitespace = (runs, before, after) => {
+	if (before !== after) {
+		pushChangedText(runs, before, after);
+		return;
+	}
+	if (before.length > 0) pushRun(runs, {
+		type: "equal",
+		before,
+		after,
+		units: 0
+	});
+};
+/**
+* Mark the whitespace a matched word gained, lost or changed, leaving the word
+* itself equal. Word tokens are aligned on their text alone, so an `equal` run
+* may pair `"Závislá"` with `" Závislá"`; both sides of such a run tokenize
+* to the same number of words. Every piece of both texts is emitted exactly
+* once and in order, so both strings reconstruct whatever the pairing.
+*/
+const separateWhitespaceChanges = (runs) => {
+	const separated = [];
+	for (const run of runs) {
+		if (run.type !== "equal" || run.before === run.after) {
+			pushRun(separated, run);
+			continue;
+		}
+		const beforeTokens = tokenizeWords(run.before);
+		const afterTokens = tokenizeWords(run.after);
+		const tokenCount = Math.max(beforeTokens.length, afterTokens.length);
+		for (let index = 0; index < tokenCount; index++) {
+			const before = splitTokenWhitespace(beforeTokens[index] ?? "");
+			const after = splitTokenWhitespace(afterTokens[index] ?? "");
+			pushWhitespace(separated, before.leading, after.leading);
+			if (before.text.length > 0 && after.text.length > 0) pushRun(separated, {
+				type: "equal",
+				before: before.text,
+				after: after.text,
+				units: 1
+			});
+			else pushChangedText(separated, before.text, after.text);
+			pushWhitespace(separated, before.trailing, after.trailing);
+		}
+	}
+	return separated;
+};
 const toSegments = (runs) => {
 	const segments = [];
 	const push = (type, text) => {
@@ -694,7 +854,13 @@ const diffWordSegmentsWithBudget = (before, after, options, budget) => {
 		text: before
 	}];
 	const granularity = options.granularity ?? "word";
-	const normalization = options.normalization ?? {};
+	const requestedNormalization = options.normalization ?? {};
+	const normalization = granularity === "word" ? {
+		...requestedNormalization,
+		whitespace: true
+	} : requestedNormalization;
+	const whitespaceIsSignificant = granularity === "word" && requestedNormalization.whitespace !== true;
+	const weighting = granularity === "word" ? MATCH_WEIGHTING.ContentFirst : MATCH_WEIGHTING.Uniform;
 	const beforeTokens = tokenize(before, granularity);
 	const afterTokens = tokenize(after, granularity);
 	if (beforeTokens.length === 0 && afterTokens.length === 0) return [];
@@ -705,12 +871,12 @@ const diffWordSegmentsWithBudget = (before, after, options, budget) => {
 		before: beforeTokens,
 		after: afterTokens,
 		normalization,
-		budget
+		budget,
+		weighting
 	});
 	let surviving = demoteRejectedMatches(aligned.runs);
-	const inventedOppositeChange = aligned.monotoneDirection === "insertion" && surviving.some(({ type }) => type === "del") || aligned.monotoneDirection === "deletion" && surviving.some(({ type }) => type === "ins");
 	const monotoneDirection = aligned.monotoneDirection;
-	if (inventedOppositeChange && monotoneDirection !== null) if (beforeTokens.length + afterTokens.length <= MAX_WORD_DIFF_ANCHOR_TOKENS && before.length + after.length <= MAX_WORD_DIFF_COMPARISON_KEY_CODE_UNITS && fitsCellBudget(beforeTokens.length, afterTokens.length, budget)) {
+	if (monotoneDirection !== null && hasOppositeChange(surviving, monotoneDirection)) if (beforeTokens.length + afterTokens.length <= MAX_WORD_DIFF_ANCHOR_TOKENS && before.length + after.length <= MAX_WORD_DIFF_COMPARISON_KEY_CODE_UNITS && fitsCellBudget(beforeTokens.length, afterTokens.length, budget)) {
 		const unanchored = [];
 		alignDenseGap({
 			before: beforeTokens,
@@ -726,12 +892,18 @@ const diffWordSegmentsWithBudget = (before, after, options, budget) => {
 				end: afterTokens.length
 			},
 			runs: unanchored,
-			budget
+			budget,
+			weighting
 		});
-		surviving = unanchored;
-	} else surviving = monotoneDirection === "insertion" && aligned.runs.some(({ type }) => type === "del") || monotoneDirection === "deletion" && aligned.runs.some(({ type }) => type === "ins") ? alignMonotoneChange(before, after, beforeTokens, afterTokens, normalization, monotoneDirection) : aligned.runs;
-	if (aligned.monotoneDirection === null && isTooFragmented(surviving, (before.length + after.length) / 2)) return wholeStringReplacement(before, after);
-	return toSegments(orderDeletionsFirst(surviving));
+		surviving = hasOppositeChange(unanchored, monotoneDirection) ? alignMonotoneChange(before, after, beforeTokens, afterTokens, normalization, monotoneDirection) : unanchored;
+	} else surviving = hasOppositeChange(aligned.runs, monotoneDirection) ? alignMonotoneChange(before, after, beforeTokens, afterTokens, normalization, monotoneDirection) : aligned.runs;
+	if (aligned.monotoneDirection === null && surviving.some(isContentChange) && isTooFragmented(surviving, (before.length + after.length) / 2)) return wholeStringReplacement(before, after);
+	let marked = whitespaceIsSignificant ? separateWhitespaceChanges(surviving) : surviving;
+	if (monotoneDirection !== null && hasOppositeChange(marked, monotoneDirection)) {
+		const monotoneRuns = alignMonotoneChange(before, after, beforeTokens, afterTokens, normalization, monotoneDirection);
+		marked = whitespaceIsSignificant ? separateWhitespaceChanges(monotoneRuns) : monotoneRuns;
+	}
+	return toSegments(orderDeletionsFirst(marked));
 };
 /**
 * One internal comparison/apply scope. Every diff shares the same quadratic
@@ -760,4 +932,4 @@ const wordDiffSessionFromOptions = (options) => {
 };
 const diffWordSegments = (before, after, options = {}) => diffWordSegmentsWithBudget(before, after, options, { remainingCells: MAX_WORD_DIFF_CELLS });
 //#endregion
-export { WORD_DIFF_GRANULARITIES, createScopedWordDiffOptions, createWordDiffSession, diffWordSegments, wordDiffSessionFromOptions };
+export { MAX_EDGE_PUNCTUATION_TOKENS, WORD_DIFF_GRANULARITIES, createScopedWordDiffOptions, createWordDiffSession, diffWordSegments, tokenizeWords, wordDiffSessionFromOptions };

package/dist/controller/fontReadiness.js

@@ -1,4 +1,5 @@
 import { buildFontAlternates, getFontAlternate } from "../fonts/fontAlternates.js";
+import { DEFAULT_FONT_FAMILY } from "../layout-engine/measure/measureHelpers.js";
 import { expectFontFamilyMarkAttrs, expectParagraphAttrs } from "../prosemirror/attrs/index.js";
 import { parseFontFamilyList, resolveFontFamily } from "../utils/fontResolver.js";
 //#region src/controller/fontReadiness.ts
@@ -11,7 +12,6 @@ function documentFontsAreLoaded() {
 	return !fontSet || fontSet.status === "loaded";
 }
 const INITIAL_LAYOUT_FONT_TIMEOUT_MS = 2e3;
-const DEFAULT_LAYOUT_FONT_FAMILY = "Calibri";
 const CSS_GENERIC_FONT_FAMILIES = /* @__PURE__ */ new Set([
 	"serif",
 	"sans-serif",
@@ -54,7 +54,7 @@ function collectInitialLayoutFontFamilies(documentModel, pmDoc) {
 function collectInitialLayoutFontFaces(documentModel, pmDoc) {
 	const faces = /* @__PURE__ */ new Map();
 	const fontAlternates = buildFontAlternates(documentModel?.package.fontTable);
-	addLayoutFontFamilyFace(faces, DEFAULT_LAYOUT_FONT_FAMILY, REGULAR_LAYOUT_FONT_DESCRIPTOR, fontAlternates);
+	addLayoutFontFamilyFace(faces, DEFAULT_FONT_FAMILY, REGULAR_LAYOUT_FONT_DESCRIPTOR, fontAlternates);
 	for (const family of documentModel?.requiredFonts ?? []) addLayoutFontFamilyFace(faces, family, REGULAR_LAYOUT_FONT_DESCRIPTOR, fontAlternates);
 	addLayoutFontFamilyFace(faces, documentModel?.package.theme?.fontScheme?.majorFont?.latin, REGULAR_LAYOUT_FONT_DESCRIPTOR, fontAlternates);
 	addLayoutFontFamilyFace(faces, documentModel?.package.theme?.fontScheme?.minorFont?.latin, REGULAR_LAYOUT_FONT_DESCRIPTOR, fontAlternates);
@@ -75,7 +75,7 @@ function collectProseMirrorFontFaces(faces, node, inheritedTextFormatting, fontA
 	if (node.type.name === "paragraph") addTextFormattingFontFaces(faces, expectParagraphAttrs(node).listMarkerFormatting, fontAlternates);
 	if (node.isText) {
 		const descriptor = layoutDescriptorFromFormattingAndMarks(textFormatting, node.marks);
-		addLayoutFontFamilyFace(faces, readFontFamilyMarkAttrs(node.marks) ?? textFormatting?.fontFamily ?? DEFAULT_LAYOUT_FONT_FAMILY, descriptor, fontAlternates);
+		addLayoutFontFamilyFace(faces, readFontFamilyMarkAttrs(node.marks) ?? textFormatting?.fontFamily ?? "Times New Roman", descriptor, fontAlternates);
 	}
 	node.forEach((child) => {
 		collectProseMirrorFontFaces(faces, child, textFormatting, fontAlternates);

package/dist/controller/hyphenationReadiness.d.ts

@@ -0,0 +1,19 @@
+import { HyphenationDictionaryError, HyphenationDictionaryId } from "../layout-engine/measure/hyphenationDictionaries.js";
+//#region src/controller/hyphenationReadiness.d.ts
+type HyphenationReadiness = {
+  /** Follow up on the dictionaries one layout run lacked. */
+  track: (missing: ReadonlySet<HyphenationDictionaryId>) => void;
+  /**
+   * Ignore loads still pending, as on unmount. A later `track` follows up
+   * afresh, so an adapter whose lifecycle re-attaches (React Strict Mode
+   * effects) keeps working.
+   */
+  cancel: () => void;
+};
+type HyphenationReadinessOptions = {
+  relayout: () => void;
+  onError: (error: HyphenationDictionaryError) => void;
+};
+declare const createHyphenationReadiness: ({ relayout, onError }: HyphenationReadinessOptions) => HyphenationReadiness;
+//#endregion
+export { HyphenationReadiness, HyphenationReadinessOptions, createHyphenationReadiness };

package/dist/controller/hyphenationReadiness.js

@@ -0,0 +1,47 @@
+import { requestHyphenationDictionary } from "../layout-engine/measure/hyphenationDictionaries.js";
+//#region src/controller/hyphenationReadiness.ts
+/**
+* Per-editor follow-up for hyphenation dictionaries a layout run lacked.
+*
+* Framework-neutral so both adapters share one implementation. The layout
+* pipeline hands {@link HyphenationReadiness.track} the dictionaries its run
+* requested but could not use; this editor then re-runs layout when one loads
+* (the load has already invalidated measured paragraphs) and reports a failed
+* one once. Editors that never asked for a dictionary hear nothing about it,
+* and an editor mounted after a failure still learns of it on its first run.
+*/
+const createHyphenationReadiness = ({ relayout, onError }) => {
+	let epoch = 0;
+	const pending = /* @__PURE__ */ new Set();
+	const reported = /* @__PURE__ */ new Set();
+	const follow = async (dictionary) => {
+		const startedIn = epoch;
+		pending.add(dictionary);
+		const settled = await requestHyphenationDictionary(dictionary);
+		if (startedIn !== epoch) return;
+		pending.delete(dictionary);
+		switch (settled.status) {
+			case "loaded":
+				relayout();
+				return;
+			case "failed":
+				if (!reported.has(dictionary)) {
+					reported.add(dictionary);
+					onError(settled.error);
+				}
+				return;
+			default:
+		}
+	};
+	return {
+		track: (missing) => {
+			for (const dictionary of missing) if (!pending.has(dictionary)) follow(dictionary);
+		},
+		cancel: () => {
+			epoch += 1;
+			pending.clear();
+		}
+	};
+};
+//#endregion
+export { createHyphenationReadiness };

package/dist/controller/layoutPipeline.d.ts

@@ -3,6 +3,7 @@ import { TemplatePreviewEntry, TemplatePreviewHiddenRange } from "../prosemirror
 import { ColumnLayout, FlowBlock, FootnoteContent, HeaderFooterContent, Layout, Measure, PageHeaderFooterRefs, PageMargins } from "../layout-engine/types.js";
 import { LayoutRunReason } from "../layout-engine/layoutInstrumentation.js";
 import { DirtyRange } from "../paged-layout/incrementalMeasure.js";
+import { HyphenationReadiness } from "./hyphenationReadiness.js";
 import { ConvertHeaderFooterOptions, HeaderFooterMetrics } from "../layout-bridge/convert/headerFooterLayout.js";
 import "../layout-engine/index.js";
 import { FootnoteRenderItem } from "../layout-painter/renderPage.js";
@@ -74,6 +75,8 @@ type LayoutPipelineDeps<THfPMs> = {
   pageRenderer?: PageRendererName;
   emptyTemplatePreviewEntries: readonly TemplatePreviewEntry[];
   emptyTemplatePreviewHidden: readonly TemplatePreviewHiddenRange[];
+  /** Follows up on the hyphenation dictionaries a run lacked (relayout or error). */
+  hyphenationReadiness: HyphenationReadiness;
 };
 declare function runLayoutPipeline<THfPMs>(deps: LayoutPipelineDeps<THfPMs>, state: EditorState, options?: LayoutRunOptions): LayoutOutcome;
 //#endregion