@pierre/diffs 1.1.8 → 1.1.9

package/dist/worker/worker.js

@@ -76,6 +76,14 @@ function createTextNodeElement(value) {
 		value
 	};
 }
+function createHastElement({ tagName, children = [], properties = {} }) {
+	return {
+		type: "element",
+		tagName,
+		properties,
+		children
+	};
+}
 function findCodeElement(nodes) {
 	let firstChild = nodes.children[0];
 	while (firstChild != null) {
@@ -107,9 +115,87 @@ function processLine(node, line, state) {
 	return node;
 }
 
+//#endregion
+//#region src/utils/wrapTokenFragments.ts
+const NO_TOKEN = Symbol("no-token");
+const MULTIPLE_TOKENS = Symbol("multiple-tokens");
+function wrapTokenFragments(container) {
+	const ownTokenChar = getTokenChar(container);
+	if (ownTokenChar != null) return ownTokenChar;
+	let containerTokenState = NO_TOKEN;
+	const wrappedChildren = [];
+	let currentTokenChildren = [];
+	let currentTokenChar;
+	const flushTokenChildren = () => {
+		if (currentTokenChildren.length === 0 || currentTokenChar == null) {
+			currentTokenChildren = [];
+			currentTokenChar = void 0;
+			return;
+		}
+		if (currentTokenChildren.length === 1) {
+			const child = currentTokenChildren[0];
+			if (child?.type === "element") {
+				setTokenChar(child, currentTokenChar);
+				for (const grandChild of child.children) stripTokenChar(grandChild);
+			} else stripTokenChar(child);
+			wrappedChildren.push(child);
+			currentTokenChildren = [];
+			currentTokenChar = void 0;
+			return;
+		}
+		for (const child of currentTokenChildren) stripTokenChar(child);
+		wrappedChildren.push(createHastElement({
+			tagName: "span",
+			properties: { "data-char": currentTokenChar },
+			children: currentTokenChildren
+		}));
+		currentTokenChildren = [];
+		currentTokenChar = void 0;
+	};
+	const mergeContainerTokenState = (childTokenState) => {
+		if (childTokenState === NO_TOKEN) return;
+		if (childTokenState === MULTIPLE_TOKENS) {
+			containerTokenState = MULTIPLE_TOKENS;
+			return;
+		}
+		if (containerTokenState === NO_TOKEN) {
+			containerTokenState = childTokenState;
+			return;
+		}
+		if (containerTokenState !== childTokenState) containerTokenState = MULTIPLE_TOKENS;
+	};
+	for (const child of container.children) {
+		const childTokenState = child.type === "element" ? wrapTokenFragments(child) : NO_TOKEN;
+		mergeContainerTokenState(childTokenState);
+		if (typeof childTokenState !== "number") {
+			flushTokenChildren();
+			wrappedChildren.push(child);
+			continue;
+		}
+		if (currentTokenChar != null && currentTokenChar !== childTokenState) flushTokenChildren();
+		currentTokenChar ??= childTokenState;
+		currentTokenChildren.push(child);
+	}
+	flushTokenChildren();
+	container.children = wrappedChildren;
+	return containerTokenState;
+}
+function getTokenChar(node) {
+	const value = node.properties["data-char"];
+	if (typeof value === "number") return value;
+}
+function stripTokenChar(node) {
+	if (node.type !== "element") return;
+	node.properties["data-char"] = void 0;
+	for (const child of node.children) stripTokenChar(child);
+}
+function setTokenChar(node, char) {
+	node.properties["data-char"] = char;
+}
+
 //#endregion
 //#region src/utils/createTransformerWithState.ts
-function createTransformerWithState(useCSSClasses = false) {
+function createTransformerWithState(useTokenTransformer = false, useCSSClasses = false) {
 	const state = { lineInfo: [] };
 	const transformers = [{
 		line(node) {
@@ -123,13 +209,37 @@ function createTransformerWithState(useCSSClasses = false) {
 				let index = 1;
 				for (const node of code.children) {
 					if (node.type !== "element") continue;
+					if (useTokenTransformer) wrapTokenFragments(node);
 					children.push(processLine(node, index, state));
 					index++;
 				}
 				code.children = children;
 			}
 			return pre;
-		}
+		},
+		...useTokenTransformer ? {
+			tokens(lines) {
+				for (const line of lines) {
+					let col = 0;
+					for (const token of line) {
+						const tokenWithOriginalRange = token;
+						tokenWithOriginalRange.__lineChar ??= col;
+						col += token.content.length;
+					}
+				}
+			},
+			preprocess(_code, options) {
+				options.mergeWhitespaces = "never";
+			},
+			span(hast, _line, _char, _lineElement, token) {
+				if (token?.offset != null && token.content != null) {
+					const tokenChar = token.__lineChar;
+					if (tokenChar != null) hast.properties["data-char"] = tokenChar;
+					return hast;
+				}
+				return hast;
+			}
+		} : null
 	}];
 	if (useCSSClasses) transformers.push(tokenStyleNormalizer, toClass);
 	return {
@@ -1191,7 +1301,7 @@ function createBucket() {
 function renderTwoFiles({ deletionFile, additionFile, deletionInfo, additionInfo, highlighter: highlighter$1, deletionDecorations, additionDecorations, languageOverride, options: { theme: themeOrThemes = DEFAULT_THEMES,...options } }) {
 	const deletionLang = languageOverride ?? getFiletypeFromFileName(deletionFile.name);
 	const additionLang = languageOverride ?? getFiletypeFromFileName(additionFile.name);
-	const { state, transformers } = createTransformerWithState();
+	const { state, transformers } = createTransformerWithState(options.useTokenTransformer);
 	const hastConfig = (() => {
 		return typeof themeOrThemes === "string" ? {
 			...options,
@@ -1292,7 +1402,7 @@ function splitFileContents(contents) {
 //#endregion
 //#region src/utils/renderFileWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS = { forcePlainText: false };
-function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES, tokenizeMaxLineLength }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
+function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -1301,7 +1411,7 @@ function renderFileWithHighlighter(file, highlighter$1, { theme = DEFAULT_THEMES
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
-	const { state, transformers } = createTransformerWithState();
+	const { state, transformers } = createTransformerWithState(useTokenTransformer);
 	const lang = forcePlainText ? "text" : file.lang ?? getFiletypeFromFileName(file.name);
 	const baseThemeType = typeof theme === "string" ? highlighter$1.getTheme(theme).type : void 0;
 	const themeStyles = getHighlighterThemeStyles({
@@ -1358,6 +1468,7 @@ function extractWindowedFileContent(lines, startingLine, totalLines) {
 let highlighter;
 let renderOptions = {
 	theme: DEFAULT_THEMES,
+	useTokenTransformer: false,
 	tokenizeMaxLineLength: 1e3,
 	lineDiffType: "word-alt",
 	maxLineDiffLength: 1e3
@@ -1421,6 +1532,7 @@ async function handleRenderFile({ id, file, resolvedLanguages }) {
 	if (resolvedLanguages != null) attachResolvedLanguages(resolvedLanguages, highlighter$1);
 	const fileOptions = {
 		theme: renderOptions.theme,
+		useTokenTransformer: renderOptions.useTokenTransformer,
 		tokenizeMaxLineLength: renderOptions.tokenizeMaxLineLength
 	};
 	sendFileSuccess(id, renderFileWithHighlighter(file, highlighter$1, fileOptions), fileOptions);