@pierre/diffs 1.1.7 → 1.1.9

package/dist/worker/worker-portable.js

@@ -14421,9 +14421,106 @@ function processLine(node, line, state) {
 	return node;
 }
 
+//#endregion
+//#region src/utils/wrapTokenFragments.ts
+const NO_TOKEN = Symbol("no-token");
+const MULTIPLE_TOKENS = Symbol("multiple-tokens");
+function wrapTokenFragments(container) {
+	const ownTokenChar = getTokenChar(container);
+	if (ownTokenChar != null) {
+		return ownTokenChar;
+	}
+	let containerTokenState = NO_TOKEN;
+	const wrappedChildren = [];
+	let currentTokenChildren = [];
+	let currentTokenChar;
+	const flushTokenChildren = () => {
+		if (currentTokenChildren.length === 0 || currentTokenChar == null) {
+			currentTokenChildren = [];
+			currentTokenChar = undefined;
+			return;
+		}
+		if (currentTokenChildren.length === 1) {
+			const child = currentTokenChildren[0];
+			if (child?.type === "element") {
+				setTokenChar(child, currentTokenChar);
+				for (const grandChild of child.children) {
+					stripTokenChar(grandChild);
+				}
+			} else {
+				stripTokenChar(child);
+			}
+			wrappedChildren.push(child);
+			currentTokenChildren = [];
+			currentTokenChar = undefined;
+			return;
+		}
+		for (const child of currentTokenChildren) {
+			stripTokenChar(child);
+		}
+		wrappedChildren.push(createHastElement({
+			tagName: "span",
+			properties: { "data-char": currentTokenChar },
+			children: currentTokenChildren
+		}));
+		currentTokenChildren = [];
+		currentTokenChar = undefined;
+	};
+	const mergeContainerTokenState = (childTokenState) => {
+		if (childTokenState === NO_TOKEN) {
+			return;
+		}
+		if (childTokenState === MULTIPLE_TOKENS) {
+			containerTokenState = MULTIPLE_TOKENS;
+			return;
+		}
+		if (containerTokenState === NO_TOKEN) {
+			containerTokenState = childTokenState;
+			return;
+		}
+		if (containerTokenState !== childTokenState) {
+			containerTokenState = MULTIPLE_TOKENS;
+		}
+	};
+	for (const child of container.children) {
+		const childTokenState = child.type === "element" ? wrapTokenFragments(child) : NO_TOKEN;
+		mergeContainerTokenState(childTokenState);
+		if (typeof childTokenState !== "number") {
+			flushTokenChildren();
+			wrappedChildren.push(child);
+			continue;
+		}
+		if (currentTokenChar != null && currentTokenChar !== childTokenState) {
+			flushTokenChildren();
+		}
+		currentTokenChar ??= childTokenState;
+		currentTokenChildren.push(child);
+	}
+	flushTokenChildren();
+	container.children = wrappedChildren;
+	return containerTokenState;
+}
+function getTokenChar(node) {
+	const value = node.properties["data-char"];
+	if (typeof value === "number") {
+		return value;
+	}
+	return undefined;
+}
+function stripTokenChar(node) {
+	if (node.type !== "element") return;
+	node.properties["data-char"] = undefined;
+	for (const child of node.children) {
+		stripTokenChar(child);
+	}
+}
+function setTokenChar(node, char) {
+	node.properties["data-char"] = char;
+}
+
 //#endregion
 //#region src/utils/createTransformerWithState.ts
-function createTransformerWithState(useCSSClasses = false) {
+function createTransformerWithState(useTokenTransformer = false, useCSSClasses = false) {
 	const state = { lineInfo: [] };
 	const transformers = [{
 		line(node) {
@@ -14437,13 +14534,42 @@ function createTransformerWithState(useCSSClasses = false) {
 				let index = 1;
 				for (const node of code.children) {
 					if (node.type !== "element") continue;
+					if (useTokenTransformer) {
+						wrapTokenFragments(node);
+					}
 					children.push(processLine(node, index, state));
 					index++;
 				}
 				code.children = children;
 			}
 			return pre;
-		}
+		},
+		...useTokenTransformer ? {
+			tokens(lines) {
+				for (const line of lines) {
+					let col = 0;
+					for (const token$1 of line) {
+						const tokenWithOriginalRange = token$1;
+						tokenWithOriginalRange.__lineChar ??= col;
+						col += token$1.content.length;
+					}
+				}
+			},
+			preprocess(_code, options) {
+				options.mergeWhitespaces = "never";
+			},
+			span(hast, _line, _char, _lineElement, token$1) {
+				if (token$1?.offset != null && token$1.content != null) {
+					const tokenWithOriginalRange = token$1;
+					const tokenChar = tokenWithOriginalRange.__lineChar;
+					if (tokenChar != null) {
+						hast.properties["data-char"] = tokenChar;
+					}
+					return hast;
+				}
+				return hast;
+			}
+		} : null
 	}];
 	if (useCSSClasses) {
 		transformers.push(tokenStyleNormalizer, toClass);
@@ -15454,6 +15580,7 @@ function renderDiffWithHighlighter(diff, highlighter$1, options, { forcePlainTex
 		deletionLines: [],
 		additionLines: []
 	};
+	const { maxLineDiffLength } = options;
 	const shouldGroupAll = !forcePlainText && !diff.isPartial;
 	const expandedHunksForIteration = forcePlainText ? expandedHunks : undefined;
 	const buckets = new Map();
@@ -15496,7 +15623,8 @@ function renderDiffWithHighlighter(diff, highlighter$1, options, { forcePlainTex
 					additionLineIndex: bucket.additionContent.length,
 					deletionDecorations: bucket.deletionDecorations,
 					additionDecorations: bucket.additionDecorations,
-					lineDiffType
+					lineDiffType,
+					maxLineDiffLength
 				});
 			}
 			if (deletionLine != null) {
@@ -15572,12 +15700,15 @@ function renderDiffWithHighlighter(diff, highlighter$1, options, { forcePlainTex
 		baseThemeType
 	};
 }
-function computeLineDiffDecorations({ deletionLine, additionLine, deletionLineIndex, additionLineIndex, deletionDecorations, additionDecorations, lineDiffType }) {
+function computeLineDiffDecorations({ deletionLine, additionLine, deletionLineIndex, additionLineIndex, deletionDecorations, additionDecorations, lineDiffType, maxLineDiffLength }) {
 	if (deletionLine == null || additionLine == null || lineDiffType === "none") {
 		return;
 	}
 	deletionLine = cleanLastNewline(deletionLine);
 	additionLine = cleanLastNewline(additionLine);
+	if (deletionLine.length > maxLineDiffLength || additionLine.length > maxLineDiffLength) {
+		return;
+	}
 	const lineDiff$1 = lineDiffType === "char" ? diffChars(deletionLine, additionLine) : diffWordsWithSpace(deletionLine, additionLine);
 	const deletionSpans = [];
 	const additionSpans = [];
@@ -15668,7 +15799,7 @@ function createBucket() {
 function renderTwoFiles({ deletionFile, additionFile, deletionInfo, additionInfo, highlighter: highlighter$1, deletionDecorations, additionDecorations, languageOverride, options: { theme: themeOrThemes = DEFAULT_THEMES,...options } }) {
 	const deletionLang = languageOverride ?? getFiletypeFromFileName(deletionFile.name);
 	const additionLang = languageOverride ?? getFiletypeFromFileName(additionFile.name);
-	const { state, transformers } = createTransformerWithState();
+	const { state, transformers } = createTransformerWithState(options.useTokenTransformer);
 	const hastConfig = (() => {
 		return typeof themeOrThemes === "string" ? {
 			...options,
@@ -15779,7 +15910,7 @@ function splitFileContents(contents) {
 //#endregion
 //#region src/utils/renderFileWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS = { forcePlainText: false };
-function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES, tokenizeMaxLineLength }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
+function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -15788,7 +15919,7 @@ function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
-	const { state, transformers } = createTransformerWithState();
+	const { state, transformers } = createTransformerWithState(useTokenTransformer);
 	const lang = forcePlainText ? "text" : file.lang ?? getFiletypeFromFileName(file.name);
 	const baseThemeType = typeof theme === "string" ? highlighter$1.getTheme(theme).type : undefined;
 	const themeStyles = getHighlighterThemeStyles({
@@ -15849,8 +15980,10 @@ function extractWindowedFileContent(lines, startingLine, totalLines) {
 let highlighter;
 let renderOptions = {
 	theme: DEFAULT_THEMES,
+	useTokenTransformer: false,
 	tokenizeMaxLineLength: 1e3,
-	lineDiffType: "word-alt"
+	lineDiffType: "word-alt",
+	maxLineDiffLength: 1e3
 };
 self.addEventListener("error", (event) => {
 	console.error("[Shiki Worker] Unhandled error:", event.error);
@@ -15921,6 +16054,7 @@ async function handleRenderFile({ id, file, resolvedLanguages }) {
 	}
 	const fileOptions = {
 		theme: renderOptions.theme,
+		useTokenTransformer: renderOptions.useTokenTransformer,
 		tokenizeMaxLineLength: renderOptions.tokenizeMaxLineLength
 	};
 	sendFileSuccess(id, renderFileWithHighlighter(file, highlighter$1, fileOptions), fileOptions);