@pierre/diffs 1.5.2 → 1.6.0-beta.1

package/dist/worker/worker.js

@@ -1,9 +1,4 @@
-import { createHighlighterCore } from "shiki/core";
-import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
-import { createOnigurumaEngine } from "shiki/engine/oniguruma";
-import { createThemeResolver } from "@pierre/theming";
 import { diffChars, diffWordsWithSpace } from "diff";
-import { transformerStyleToClass } from "@shikijs/transformers";
 import { normalizeThemeColors } from "@pierre/theming/color";
 (() => {
 	try {
@@ -23,52 +18,78 @@ Object.freeze({
 //#endregion
 //#region src/highlighter/languages/constants.ts
 const ResolvedLanguages = /* @__PURE__ */ new Map();
+const ResolvingLanguages = /* @__PURE__ */ new Map();
+const RegisteredCustomLanguages = /* @__PURE__ */ new Map();
 const AttachedLanguages = /* @__PURE__ */ new Set();
 //#endregion
 //#region src/highlighter/languages/attachResolvedLanguages.ts
+/**
+* Record resolved grammars in the shared language cache and attach them to a
+* highlighter. Highlights bundles its lexers and ignores the grammars.
+*/
 function attachResolvedLanguages(resolvedLanguages, highlighter) {
-	resolvedLanguages = Array.isArray(resolvedLanguages) ? resolvedLanguages : [resolvedLanguages];
-	for (const resolvedLang of resolvedLanguages) {
-		if (AttachedLanguages.has(resolvedLang.name)) continue;
+	for (const resolvedLang of Array.isArray(resolvedLanguages) ? resolvedLanguages : [resolvedLanguages]) {
 		let lang = ResolvedLanguages.get(resolvedLang.name);
 		if (lang == null) {
 			lang = resolvedLang;
 			ResolvedLanguages.set(resolvedLang.name, lang);
 		}
-		const grammar = lang.data.find((grammar) => grammar.name === lang.name || grammar.aliases?.includes(lang.name) === true);
-		if (grammar == null) throw new Error(`attachResolvedLanguages: No returned grammar declares "${lang.name}" as its name or an alias.`);
-		highlighter.loadLanguageSync(lang.data);
-		try {
-			highlighter.getLanguage(lang.name);
-		} catch {
-			throw new Error(`attachResolvedLanguages: "${grammar.name}" is already loaded without alias "${lang.name}". Load the alias first or give the grammar a unique name.`);
-		}
+		highlighter.attachLanguages([lang]);
 		AttachedLanguages.add(lang.name);
 	}
 }
 //#endregion
-//#region src/highlighter/themes/constants.ts
-const AttachedThemes = /* @__PURE__ */ new Set();
+//#region src/highlighter/highlighterType.ts
+let activeType;
+let holds = 0;
+var HighlighterDisposedError = class extends Error {
+	constructor() {
+		super("Highlighter is disposed");
+		this.name = "HighlighterDisposedError";
+	}
+};
+function resolveHighlighterType(type) {
+	return type ?? activeType ?? "shiki-js";
+}
+function assertHighlighterType(type) {
+	if (activeType != null && activeType !== type) throw new Error(`Cannot load the "${type}" highlighter while "${activeType}" is in use. Only one highlighter type can be active at a time; dispose every "${activeType}" instance (disposeHighlighter() disposes the shared one) before switching.`);
+}
+function acquireHighlighterType(type) {
+	assertHighlighterType(type);
+	activeType = type;
+	holds++;
+}
+function releaseHighlighterType() {
+	holds--;
+	if (holds === 0) activeType = void 0;
+}
 //#endregion
-//#region src/highlighter/themes/themeResolver.ts
-const themeResolver = createThemeResolver();
+//#region src/highlighter/shared_highlighter.ts
+/**
+* Creates an instance the caller must dispose. Defaults to the active type
+* or DEFAULT_HIGHLIGHTER; rejects if a different type is already active.
+*/
+async function createHighlighter(type) {
+	const resolvedType = resolveHighlighterType(type);
+	acquireHighlighterType(resolvedType);
+	try {
+		if (resolvedType === "highlights") {
+			const { HighlightsHighlighter } = await import("./highlights-sAHlxqlf.js");
+			return new HighlightsHighlighter();
+		}
+		const { ShikiHighlighter } = await import("./shiki-Bx2JpdqZ.js");
+		return await ShikiHighlighter.create(resolvedType);
+	} finally {
+		releaseHighlighterType();
+	}
+}
 //#endregion
 //#region src/highlighter/themes/attachResolvedThemes.ts
+/** Seed themes received from the main thread into the worker's backend resolver. */
 function attachResolvedThemes(themes, highlighter) {
-	themes = Array.isArray(themes) ? themes : [themes];
-	for (let themeRef of themes) {
-		let resolvedTheme;
-		if (typeof themeRef === "string") {
-			resolvedTheme = themeResolver.getResolvedTheme(themeRef);
-			if (resolvedTheme == null) throw new Error(`loadResolvedThemes: ${themeRef} is not resolved, you must resolve it before calling loadResolvedThemes`);
-		} else {
-			resolvedTheme = themeRef;
-			themeRef = themeRef.name;
-			if (themeResolver.getResolvedTheme(themeRef) == null) themeResolver.seedResolvedTheme(themeRef, resolvedTheme);
-		}
-		if (AttachedThemes.has(themeRef)) continue;
-		AttachedThemes.add(themeRef);
-		highlighter.loadThemeSync(resolvedTheme);
+	for (const theme of Array.isArray(themes) ? themes : [themes]) {
+		const resolved = typeof theme === "string" ? highlighter.themeResolver.getResolvedThemes([theme])[0] : theme;
+		highlighter.themeResolver.seedResolvedTheme(resolved.name, resolved);
 	}
 }
 //#endregion
@@ -478,210 +499,6 @@ function cleanLastNewline(contents) {
 	return contents.slice(0, end);
 }
 //#endregion
-//#region src/utils/hast_utils.ts
-function createTextNodeElement(value) {
-	return {
-		type: "text",
-		value
-	};
-}
-function createHastElement({ tagName, children = [], properties = {} }) {
-	return {
-		type: "element",
-		tagName,
-		properties,
-		children
-	};
-}
-function findCodeElement(nodes) {
-	let firstChild = nodes.children[0];
-	while (firstChild != null) {
-		if (firstChild.type === "element" && firstChild.tagName === "code") return firstChild;
-		if ("children" in firstChild) firstChild = firstChild.children[0];
-		else firstChild = null;
-	}
-}
-//#endregion
-//#region src/utils/processLine.ts
-function processLine(node, line, state) {
-	const lineInfo = typeof state.lineInfo === "function" ? state.lineInfo(line) : state.lineInfo[line - 1];
-	if (lineInfo == null) {
-		const errorMessage = `processLine: line ${line}, contains no state.lineInfo`;
-		console.error(errorMessage, {
-			node,
-			line,
-			state
-		});
-		throw new Error(errorMessage);
-	}
-	node.tagName = "div";
-	node.properties["data-line"] = lineInfo.lineNumber;
-	node.properties["data-alt-line"] = lineInfo.altLineNumber;
-	node.properties["data-line-type"] = lineInfo.type;
-	node.properties["data-line-index"] = lineInfo.lineIndex;
-	if (node.children.length === 0) node.children.push(createTextNodeElement("\n"));
-	return node;
-}
-//#endregion
-//#region src/utils/wrapTokenFragments.ts
-const NO_TOKEN = Symbol("no-token");
-const MULTIPLE_TOKENS = Symbol("multiple-tokens");
-function wrapTokenFragments(container) {
-	const ownTokenChar = getTokenChar(container);
-	if (ownTokenChar != null) return ownTokenChar;
-	let containerTokenState = NO_TOKEN;
-	const wrappedChildren = [];
-	let currentTokenChildren = [];
-	let currentTokenChar;
-	const flushTokenChildren = () => {
-		if (currentTokenChildren.length === 0 || currentTokenChar == null) {
-			currentTokenChildren = [];
-			currentTokenChar = void 0;
-			return;
-		}
-		if (currentTokenChildren.length === 1) {
-			const child = currentTokenChildren[0];
-			if (child?.type === "element") {
-				setTokenChar(child, currentTokenChar);
-				for (const grandChild of child.children) stripTokenChar(grandChild);
-			} else stripTokenChar(child);
-			wrappedChildren.push(child);
-			currentTokenChildren = [];
-			currentTokenChar = void 0;
-			return;
-		}
-		for (const child of currentTokenChildren) stripTokenChar(child);
-		wrappedChildren.push(createHastElement({
-			tagName: "span",
-			properties: { "data-char": currentTokenChar },
-			children: currentTokenChildren
-		}));
-		currentTokenChildren = [];
-		currentTokenChar = void 0;
-	};
-	const mergeContainerTokenState = (childTokenState) => {
-		if (childTokenState === NO_TOKEN) return;
-		if (childTokenState === MULTIPLE_TOKENS) {
-			containerTokenState = MULTIPLE_TOKENS;
-			return;
-		}
-		if (containerTokenState === NO_TOKEN) {
-			containerTokenState = childTokenState;
-			return;
-		}
-		if (containerTokenState !== childTokenState) containerTokenState = MULTIPLE_TOKENS;
-	};
-	for (const child of container.children) {
-		const childTokenState = child.type === "element" ? wrapTokenFragments(child) : NO_TOKEN;
-		mergeContainerTokenState(childTokenState);
-		if (typeof childTokenState !== "number") {
-			flushTokenChildren();
-			wrappedChildren.push(child);
-			continue;
-		}
-		if (currentTokenChar != null && currentTokenChar !== childTokenState) flushTokenChildren();
-		currentTokenChar ??= childTokenState;
-		currentTokenChildren.push(child);
-	}
-	flushTokenChildren();
-	container.children = wrappedChildren;
-	return containerTokenState;
-}
-function getTokenChar(node) {
-	const value = node.properties["data-char"];
-	if (typeof value === "number") return value;
-}
-function stripTokenChar(node) {
-	if (node.type !== "element") return;
-	node.properties["data-char"] = void 0;
-	for (const child of node.children) stripTokenChar(child);
-}
-function setTokenChar(node, char) {
-	node.properties["data-char"] = char;
-}
-//#endregion
-//#region src/utils/createTransformerWithState.ts
-function createTransformerWithState(useTokenTransformer = false, useCSSClasses = false) {
-	const state = { lineInfo: [] };
-	const transformers = [{
-		line(node) {
-			delete node.properties.class;
-			return node;
-		},
-		pre(pre) {
-			const code = findCodeElement(pre);
-			const children = [];
-			if (code != null) {
-				let index = 1;
-				for (const node of code.children) {
-					if (node.type !== "element") continue;
-					if (useTokenTransformer) wrapTokenFragments(node);
-					children.push(processLine(node, index, state));
-					index++;
-				}
-				code.children = children;
-			}
-			return pre;
-		},
-		...useTokenTransformer ? {
-			tokens(lines) {
-				for (const line of lines) {
-					let col = 0;
-					for (const token of line) {
-						const tokenWithOriginalRange = token;
-						tokenWithOriginalRange.__lineChar ??= col;
-						col += token.content.length;
-					}
-				}
-			},
-			preprocess(_code, options) {
-				options.mergeWhitespaces = "never";
-			},
-			span(hast, _line, _char, _lineElement, token) {
-				if (token?.offset != null && token.content != null) {
-					const tokenChar = token.__lineChar;
-					if (tokenChar != null) hast.properties["data-char"] = tokenChar;
-					return hast;
-				}
-				return hast;
-			}
-		} : null
-	}];
-	if (useCSSClasses) transformers.push(tokenStyleNormalizer, toClass);
-	if (useTokenTransformer) transformers.push({ line: (node) => {
-		if (node.type === "element" && node.children.length === 0) node.children.push({
-			type: "element",
-			tagName: "br",
-			properties: {},
-			children: []
-		});
-		return node;
-	} });
-	return {
-		state,
-		transformers,
-		toClass
-	};
-}
-const toClass = transformerStyleToClass({ classPrefix: "hl-" });
-const tokenStyleNormalizer = {
-	name: "token-style-normalizer",
-	tokens(lines) {
-		for (const line of lines) for (const token of line) {
-			if (token.htmlStyle != null) continue;
-			const style = {};
-			if (token.color != null) style.color = token.color;
-			if (token.bgColor != null) style["background-color"] = token.bgColor;
-			if (token.fontStyle != null && token.fontStyle !== 0) {
-				if ((token.fontStyle & 1) !== 0) style["font-style"] = "italic";
-				if ((token.fontStyle & 2) !== 0) style["font-weight"] = "bold";
-				if ((token.fontStyle & 4) !== 0) style["text-decoration"] = "underline";
-			}
-			if (Object.keys(style).length > 0) token.htmlStyle = style;
-		}
-	}
-};
-//#endregion
 //#region src/utils/formatCSSVariablePrefix.ts
 function formatCSSVariablePrefix(type) {
 	return `--${type === "token" ? "diffs-token" : "diffs"}-`;
@@ -692,7 +509,7 @@ function getHighlighterThemeStyles({ theme = DEFAULT_THEMES, highlighter, prefix
 	let styles = "";
 	if (typeof theme === "string") {
 		const themeData = highlighter.getTheme(theme);
-		const normalized = normalizeThemeColors(themeData);
+		const normalized = getThemeColors(themeData, highlighter);
 		styles += `color:${normalized.fg};`;
 		styles += `background-color:${normalized.bg};`;
 		styles += `${formatCSSVariablePrefix("global")}fg:${normalized.fg};`;
@@ -700,18 +517,28 @@ function getHighlighterThemeStyles({ theme = DEFAULT_THEMES, highlighter, prefix
 		styles += getGitVariables(themeData, prefix);
 	} else {
 		let themeData = highlighter.getTheme(theme.dark);
-		let normalized = normalizeThemeColors(themeData);
+		let normalized = getThemeColors(themeData, highlighter);
 		styles += `${formatCSSVariablePrefix("global")}dark:${normalized.fg};`;
 		styles += `${formatCSSVariablePrefix("global")}dark-bg:${normalized.bg};`;
 		styles += getGitVariables(themeData, "dark");
 		themeData = highlighter.getTheme(theme.light);
-		normalized = normalizeThemeColors(themeData);
+		normalized = getThemeColors(themeData, highlighter);
 		styles += `${formatCSSVariablePrefix("global")}light:${normalized.fg};`;
 		styles += `${formatCSSVariablePrefix("global")}light-bg:${normalized.bg};`;
 		styles += getGitVariables(themeData, "light");
 	}
 	return styles;
 }
+function getThemeColors(theme, highlighter) {
+	if (highlighter.name === "highlights" && theme.zed?.cssVariables === true) {
+		const prefix = formatCSSVariablePrefix("token");
+		return {
+			fg: `var(${prefix}foreground)`,
+			bg: `var(${prefix}background)`
+		};
+	}
+	return normalizeThemeColors(theme);
+}
 function getGitVariables(themeData, modePrefix) {
 	modePrefix = modePrefix != null ? `${modePrefix}-` : "";
 	let styles = "";
@@ -724,16 +551,16 @@ function getGitVariables(themeData, modePrefix) {
 	return styles;
 }
 //#endregion
-//#region src/utils/getLineNodes.ts
-function getLineNodes(nodes) {
-	let firstChild = nodes.children[0];
-	while (firstChild != null) {
-		if (firstChild.type === "element" && firstChild.tagName === "code") return firstChild.children;
-		if ("children" in firstChild) firstChild = firstChild.children[0];
-		else firstChild = null;
-	}
-	console.error(nodes);
-	throw new Error("getLineNodes: Unable to find children");
+//#region src/utils/getTokenOptions.ts
+function getTokenOptions(lang, theme, tokenizeMaxLineLength) {
+	return {
+		lang,
+		...typeof theme === "string" ? { theme } : { themes: theme },
+		defaultColor: false,
+		cssVariablePrefix: formatCSSVariablePrefix("token"),
+		tokenizeMaxLineLength,
+		tokenizeTimeLimit: 0
+	};
 }
 //#endregion
 //#region src/utils/getHunkSideBoundaries.ts
@@ -1327,9 +1154,423 @@ function pushOrJoinSpan({ item, arr, enableJoin, isNeutral = false, isLastItem =
 	arr.push([isNeutral ? 0 : 1, item.value]);
 }
 //#endregion
+//#region src/utils/hast_utils.ts
+function createTextNodeElement(value) {
+	return {
+		type: "text",
+		value
+	};
+}
+function createHastElement({ tagName, children = [], properties = {} }) {
+	return {
+		type: "element",
+		tagName,
+		properties,
+		children
+	};
+}
+//#endregion
+//#region src/utils/processLine.ts
+function processLine(node, line, state) {
+	const lineInfo = typeof state.lineInfo === "function" ? state.lineInfo(line) : state.lineInfo[line - 1];
+	if (lineInfo == null) {
+		const errorMessage = `processLine: line ${line}, contains no state.lineInfo`;
+		console.error(errorMessage, {
+			node,
+			line,
+			state
+		});
+		throw new Error(errorMessage);
+	}
+	node.tagName = "div";
+	node.properties["data-line"] = lineInfo.lineNumber;
+	node.properties["data-alt-line"] = lineInfo.altLineNumber;
+	node.properties["data-line-type"] = lineInfo.type;
+	node.properties["data-line-index"] = lineInfo.lineIndex;
+	if (node.children.length === 0) node.children.push(createTextNodeElement("\n"));
+	return node;
+}
+//#endregion
+//#region src/utils/setDeferredArrayItem.ts
+function setDeferredArrayItem(array, index, create) {
+	let initialized = false;
+	let cached;
+	Object.defineProperty(array, index, {
+		configurable: true,
+		enumerable: true,
+		get() {
+			if (!initialized) {
+				cached = create();
+				initialized = true;
+			}
+			if (Object.getOwnPropertyDescriptor(array, index)?.configurable === true) Object.defineProperty(array, index, {
+				configurable: true,
+				enumerable: true,
+				writable: true,
+				value: cached
+			});
+			return cached;
+		},
+		set(value) {
+			if (Object.getOwnPropertyDescriptor(array, index)?.configurable === true) Object.defineProperty(array, index, {
+				configurable: true,
+				enumerable: true,
+				writable: true,
+				value
+			});
+			else if (Object.isFrozen(array)) throw new TypeError(`Cannot assign to read only array index '${index}'`);
+			else {
+				cached = value;
+				initialized = true;
+			}
+		}
+	});
+}
+//#endregion
+//#region src/utils/tokenStyle.ts
+const htmlStyleCache = /* @__PURE__ */ new WeakMap();
+function tokenStyle(token, cacheHtmlStyle) {
+	const { htmlStyle } = token;
+	if (htmlStyle != null) {
+		let style = cacheHtmlStyle ? htmlStyleCache.get(htmlStyle) : void 0;
+		if (style == null) {
+			style = "";
+			for (const name in htmlStyle) style += `${style === "" ? "" : ";"}${name}:${htmlStyle[name]}`;
+			if (cacheHtmlStyle) htmlStyleCache.set(htmlStyle, style);
+		}
+		return style;
+	}
+	let style = token.color != null ? `color:${token.color}` : "";
+	if (token.bgColor != null) style += `;background-color:${token.bgColor}`;
+	if (token.fontStyle != null) {
+		if ((token.fontStyle & 1) !== 0) style += ";font-style:italic";
+		if ((token.fontStyle & 2) !== 0) style += ";font-weight:bold";
+		const underline = (token.fontStyle & 4) !== 0;
+		const strikethrough = (token.fontStyle & 8) !== 0;
+		if (underline || strikethrough) style += `;text-decoration:${underline ? "underline" : ""}${underline && strikethrough ? " " : ""}${strikethrough ? "line-through" : ""}`;
+	}
+	return style.replace(/^;/, "");
+}
+//#endregion
+//#region src/utils/wrapTokenFragments.ts
+const NO_TOKEN = Symbol("no-token");
+const MULTIPLE_TOKENS = Symbol("multiple-tokens");
+function wrapTokenFragments(container) {
+	const ownTokenChar = getTokenChar(container);
+	if (ownTokenChar != null) return ownTokenChar;
+	let containerTokenState = NO_TOKEN;
+	const wrappedChildren = [];
+	let currentTokenChildren = [];
+	let currentTokenChar;
+	const flushTokenChildren = () => {
+		if (currentTokenChildren.length === 0 || currentTokenChar == null) {
+			currentTokenChildren = [];
+			currentTokenChar = void 0;
+			return;
+		}
+		if (currentTokenChildren.length === 1) {
+			const child = currentTokenChildren[0];
+			if (child?.type === "element") {
+				setTokenChar(child, currentTokenChar);
+				for (const grandChild of child.children) stripTokenChar(grandChild);
+			} else stripTokenChar(child);
+			wrappedChildren.push(child);
+			currentTokenChildren = [];
+			currentTokenChar = void 0;
+			return;
+		}
+		for (const child of currentTokenChildren) stripTokenChar(child);
+		wrappedChildren.push(createHastElement({
+			tagName: "span",
+			properties: { "data-char": currentTokenChar },
+			children: currentTokenChildren
+		}));
+		currentTokenChildren = [];
+		currentTokenChar = void 0;
+	};
+	const mergeContainerTokenState = (childTokenState) => {
+		if (childTokenState === NO_TOKEN) return;
+		if (childTokenState === MULTIPLE_TOKENS) {
+			containerTokenState = MULTIPLE_TOKENS;
+			return;
+		}
+		if (containerTokenState === NO_TOKEN) {
+			containerTokenState = childTokenState;
+			return;
+		}
+		if (containerTokenState !== childTokenState) containerTokenState = MULTIPLE_TOKENS;
+	};
+	for (const child of container.children) {
+		const childTokenState = child.type === "element" ? wrapTokenFragments(child) : NO_TOKEN;
+		mergeContainerTokenState(childTokenState);
+		if (typeof childTokenState !== "number") {
+			flushTokenChildren();
+			wrappedChildren.push(child);
+			continue;
+		}
+		if (currentTokenChar != null && currentTokenChar !== childTokenState) flushTokenChildren();
+		currentTokenChar ??= childTokenState;
+		currentTokenChildren.push(child);
+	}
+	flushTokenChildren();
+	container.children = wrappedChildren;
+	return containerTokenState;
+}
+function getTokenChar(node) {
+	const value = node.properties["data-char"];
+	if (typeof value === "number") return value;
+}
+function stripTokenChar(node) {
+	if (node.type !== "element") return;
+	node.properties["data-char"] = void 0;
+	for (const child of node.children) stripTokenChar(child);
+}
+function setTokenChar(node, char) {
+	node.properties["data-char"] = char;
+}
+//#endregion
+//#region src/utils/renderTokenLines.ts
+function renderTokenLines(lines, { state, lazyLineAST = false, cacheHtmlStyles = false, useTokenTransformer = false, mergeWhitespaces = "always", decorations = [], lineOffsets = [] }) {
+	let decorationsByLine;
+	for (const decoration of decorations) {
+		const byLine = decorationsByLine ??= /* @__PURE__ */ new Map();
+		const normalized = {
+			...decoration,
+			start: getDecorationPosition(decoration.start, lineOffsets),
+			end: getDecorationPosition(decoration.end, lineOffsets)
+		};
+		const lastLine = Math.min(normalized.end.line, lines.length - 1);
+		for (let line = normalized.start.line; line <= lastLine; line++) {
+			let spans = byLine.get(line);
+			if (spans == null) byLine.set(line, spans = []);
+			spans.push(normalized);
+		}
+	}
+	const renderLine = (tokens, lineIndex) => {
+		let normalized;
+		if (useTokenTransformer || mergeWhitespaces !== "never") {
+			let pendingWhitespace = "";
+			let whitespaceOffset = 0;
+			for (let i = 0; i < tokens.length; i++) {
+				let token = tokens[i];
+				if (useTokenTransformer) {
+					const content = token.content.trim();
+					if (content !== "" && content !== token.content) {
+						normalized ??= tokens.slice(0, i);
+						const start = token.content.indexOf(content);
+						if (start > 0) normalized.push({
+							content: token.content.slice(0, start),
+							offset: token.offset
+						});
+						normalized.push({
+							...token,
+							content,
+							offset: token.offset + start
+						});
+						const end = start + content.length;
+						if (end < token.content.length) normalized.push({
+							content: token.content.slice(end),
+							offset: token.offset + end
+						});
+						continue;
+					}
+				} else {
+					const decoratedWhitespace = ((token.fontStyle ?? 0) & 12) !== 0;
+					if (!decoratedWhitespace && /^\s+$/.test(token.content) && i < tokens.length - 1) {
+						normalized ??= tokens.slice(0, i);
+						if (pendingWhitespace === "") whitespaceOffset = token.offset;
+						pendingWhitespace += token.content;
+						continue;
+					}
+					if (pendingWhitespace !== "" && normalized != null) {
+						if (decoratedWhitespace) normalized.push({
+							content: pendingWhitespace,
+							offset: whitespaceOffset
+						});
+						else token = {
+							...token,
+							content: pendingWhitespace + token.content,
+							offset: whitespaceOffset
+						};
+						pendingWhitespace = "";
+					}
+				}
+				normalized?.push(token);
+			}
+		}
+		normalized ??= tokens;
+		const line = {
+			type: "element",
+			tagName: state == null ? "span" : "div",
+			properties: state == null ? { class: "line" } : {},
+			children: []
+		};
+		const spans = decorationsByLine?.get(lineIndex);
+		const column = spans == null ? appendTokens(line, normalized, useTokenTransformer, cacheHtmlStyles) : appendDecoratedTokens(line, normalized, spans, lineIndex, useTokenTransformer, cacheHtmlStyles);
+		if (useTokenTransformer && column === 0) line.children.push({
+			type: "element",
+			tagName: "br",
+			properties: {},
+			children: []
+		});
+		if (useTokenTransformer && spans != null) wrapTokenFragments(line);
+		return state == null ? line : processLine(line, lineIndex + 1, state);
+	};
+	if (!lazyLineAST) return lines.map(renderLine);
+	const rows = new Array(lines.length);
+	for (let index = 0; index < lines.length; index++) {
+		const tokens = lines[index];
+		setDeferredArrayItem(rows, index, () => renderLine(tokens, index));
+	}
+	return rows;
+}
+function appendTokens(line, tokens, useTokenTransformer, cacheHtmlStyles) {
+	let column = 0;
+	for (const token of tokens) {
+		if (token.content === "") continue;
+		line.children.push(createTokenSpan(token, tokenStyle(token, cacheHtmlStyles), token.content, column, useTokenTransformer));
+		column += token.content.length;
+	}
+	return column;
+}
+function appendDecoratedTokens(line, tokens, spans, lineIndex, useTokenTransformer, cacheHtmlStyles) {
+	if (spans.length > 1) spans.sort((a, b) => {
+		if (a.start.line !== b.start.line) return a.start.line - b.start.line;
+		if (a.start.character !== b.start.character) return a.start.character - b.start.character;
+		if (a.end.line !== b.end.line) return b.end.line - a.end.line;
+		return b.end.character - a.end.character;
+	});
+	const lineLength = tokens.reduce((length, token) => length + token.content.length, 0);
+	const boundaries = [];
+	for (let order = 0; order < spans.length; order++) {
+		const decoration = spans[order];
+		const range = {
+			decoration,
+			order,
+			from: decoration.start.line === lineIndex ? decoration.start.character : 0,
+			to: decoration.end.line === lineIndex ? decoration.end.character : lineLength
+		};
+		boundaries.push({
+			position: Math.max(0, Math.min(range.from, lineLength)),
+			range,
+			start: true
+		}, {
+			position: Math.max(0, Math.min(range.to, lineLength)),
+			range,
+			start: false
+		});
+	}
+	boundaries.sort((a, b) => a.position - b.position);
+	const active = [];
+	const decorationStack = [];
+	let boundaryIndex = 0;
+	const updateActive = (range, add) => {
+		let low = 0;
+		let high = active.length;
+		while (low < high) {
+			const middle = low + high >>> 1;
+			if (active[middle].order < range.order) low = middle + 1;
+			else high = middle;
+		}
+		if (add) active.splice(low, 0, range);
+		else if (active[low] === range) active.splice(low, 1);
+	};
+	const getParent = () => {
+		let parent = line;
+		let depth = 0;
+		for (const range of active) {
+			if (decorationStack[depth]?.range !== range) {
+				decorationStack.length = depth;
+				const wrapper = {
+					type: "element",
+					tagName: "span",
+					properties: { ...range.decoration.properties },
+					children: []
+				};
+				parent.children.push(wrapper);
+				decorationStack.push({
+					range,
+					node: wrapper
+				});
+			}
+			parent = decorationStack[depth++].node;
+		}
+		decorationStack.length = depth;
+		return parent;
+	};
+	let parent = line;
+	const advanceBoundary = (position) => {
+		const first = boundaryIndex;
+		while (boundaries[boundaryIndex]?.position === position) boundaryIndex++;
+		let hasMarker = false;
+		for (let i = first; i < boundaryIndex; i++) {
+			const { range, start } = boundaries[i];
+			if (!start && range.from !== range.to && position < lineLength) updateActive(range, false);
+		}
+		for (let i = first; i < boundaryIndex; i++) {
+			const { range, start } = boundaries[i];
+			if (start && range.from <= position && range.to >= position && (range.to > position || range.from === position || position === lineLength)) {
+				updateActive(range, true);
+				if (range.from === range.to) hasMarker = true;
+			}
+		}
+		if (hasMarker) {
+			getParent();
+			for (let i = first; i < boundaryIndex; i++) {
+				const { range, start } = boundaries[i];
+				if (start && range.from === range.to) updateActive(range, false);
+			}
+		}
+		if (position < lineLength) parent = getParent();
+	};
+	let column = 0;
+	for (const token of tokens) {
+		if (token.content === "") continue;
+		const start = column;
+		const end = start + token.content.length;
+		const style = tokenStyle(token, cacheHtmlStyles);
+		while (column < end) {
+			if (boundaries[boundaryIndex]?.position === column) advanceBoundary(column);
+			const to = Math.min(end, boundaries[boundaryIndex]?.position ?? end);
+			parent.children.push(createTokenSpan(token, style, column === start && to === end ? token.content : token.content.slice(column - start, to - start), start, useTokenTransformer));
+			column = to;
+		}
+	}
+	if (boundaries[boundaryIndex]?.position === column) advanceBoundary(column);
+	return column;
+}
+function createTokenSpan(token, style, text, start, useTokenTransformer) {
+	const properties = token.htmlAttrs != null ? { ...token.htmlAttrs } : {};
+	if (style !== "") properties.style = style;
+	if (useTokenTransformer) properties["data-char"] = start;
+	return {
+		type: "element",
+		tagName: "span",
+		properties,
+		children: [{
+			type: "text",
+			value: text
+		}]
+	};
+}
+function getDecorationPosition(position, lineOffsets) {
+	if (typeof position !== "number") return position;
+	let low = 0;
+	let high = lineOffsets.length;
+	while (low + 1 < high) {
+		const middle = low + high >>> 1;
+		if (lineOffsets[middle] <= position) low = middle;
+		else high = middle;
+	}
+	return {
+		line: low,
+		character: position - (lineOffsets[low] ?? 0)
+	};
+}
+//#endregion
 //#region src/utils/renderDiffWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS$1 = { forcePlainText: false };
-function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText, startingLine, totalLines, expandedHunks, collapsedContextThreshold = 1 } = DEFAULT_PLAIN_TEXT_OPTIONS$1) {
+function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText, lazyLineAST = false, startingLine, totalLines, expandedHunks, collapsedContextThreshold = 1 } = DEFAULT_PLAIN_TEXT_OPTIONS$1) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -1338,6 +1579,7 @@ function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText,
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
+	lazyLineAST &&= !forcePlainText && !options.useTokenTransformer;
 	const baseThemeType = typeof options.theme === "string" ? highlighter.getTheme(options.theme).type : void 0;
 	const themeStyles = getHighlighterThemeStyles({
 		theme: options.theme,
@@ -1432,17 +1674,27 @@ function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText,
 			additionDecorations: bucket.additionDecorations,
 			highlighter,
 			options,
-			languageOverride: forcePlainText ? "text" : diff.lang
+			languageOverride: forcePlainText ? "text" : diff.lang,
+			lazyLineAST
 		});
 		if (shouldGroupAll) {
 			code.deletionLines = deletionLines;
 			code.additionLines = additionLines;
 			continue;
 		}
-		if (bucket.deletionSegments.length > 0) for (const seg of bucket.deletionSegments) for (let i = 0; i < seg.count; i++) code.deletionLines[seg.targetIndex + i] = deletionLines[seg.originalOffset + i];
-		else appendItems(code.deletionLines, deletionLines);
-		if (bucket.additionSegments.length > 0) for (const seg of bucket.additionSegments) for (let i = 0; i < seg.count; i++) code.additionLines[seg.targetIndex + i] = additionLines[seg.originalOffset + i];
-		else appendItems(code.additionLines, additionLines);
+		for (const [target, lines, segments] of [[
+			code.deletionLines,
+			deletionLines,
+			bucket.deletionSegments
+		], [
+			code.additionLines,
+			additionLines,
+			bucket.additionSegments
+		]]) if (segments.length > 0) for (const segment of segments) for (let index = 0; index < segment.count; index++) target[segment.targetIndex + index] = lines[segment.originalOffset + index];
+		else if (lazyLineAST) {
+			const offset = target.length;
+			for (let index = 0; index < lines.length; index++) setDeferredArrayItem(target, offset + index, () => lines[index]);
+		} else appendItems(target, lines);
 	}
 	return {
 		code,
@@ -1574,46 +1826,20 @@ function createBucket() {
 		additionSegments: []
 	};
 }
-function renderTwoFiles({ deletionFile, additionFile, deletionInfo, additionInfo, highlighter, deletionDecorations, additionDecorations, languageOverride, options: { theme: themeOrThemes, ...options } }) {
-	const deletionLang = languageOverride ?? getFiletypeFromFileName(deletionFile.name);
-	const additionLang = languageOverride ?? getFiletypeFromFileName(additionFile.name);
-	const { state, transformers } = createTransformerWithState(options.useTokenTransformer);
-	const hastConfig = (() => {
-		return typeof themeOrThemes === "string" ? {
-			...options,
-			lang: "text",
-			theme: themeOrThemes,
-			transformers,
-			decorations: void 0,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		} : {
-			...options,
-			lang: "text",
-			themes: themeOrThemes,
-			transformers,
-			decorations: void 0,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		};
-	})();
+function renderTwoFiles({ lazyLineAST, deletionFile, additionFile, deletionInfo, additionInfo, highlighter, deletionDecorations, additionDecorations, languageOverride, options: { theme, useTokenTransformer, tokenizeMaxLineLength } }) {
+	const renderFile = (file, lineInfo, decorations) => {
+		if (file.contents === "") return [];
+		return renderTokenLines(highlighter.codeToTokens(cleanLastNewline(file.contents), getTokenOptions(languageOverride ?? getFiletypeFromFileName(file.name), theme, tokenizeMaxLineLength)).tokens, {
+			state: { lineInfo },
+			useTokenTransformer,
+			decorations,
+			lazyLineAST,
+			cacheHtmlStyles: highlighter.name === "highlights"
+		});
+	};
 	return {
-		deletionLines: (() => {
-			if (deletionFile.contents === "") return [];
-			hastConfig.lang = deletionLang;
-			state.lineInfo = deletionInfo;
-			hastConfig.decorations = deletionDecorations;
-			return getLineNodes(highlighter.codeToHast(cleanLastNewline(deletionFile.contents), hastConfig));
-		})(),
-		additionLines: (() => {
-			if (additionFile.contents === "") return [];
-			hastConfig.lang = additionLang;
-			hastConfig.decorations = additionDecorations;
-			state.lineInfo = additionInfo;
-			return getLineNodes(highlighter.codeToHast(cleanLastNewline(additionFile.contents), hastConfig));
-		})()
+		deletionLines: renderFile(deletionFile, deletionInfo, deletionDecorations),
+		additionLines: renderFile(additionFile, additionInfo, additionDecorations)
 	};
 }
 //#endregion
@@ -1679,7 +1905,7 @@ function linesFromFileContents(contents) {
 //#endregion
 //#region src/utils/renderFileWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS = { forcePlainText: false };
-function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
+function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, lazyLineAST = false, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -1688,7 +1914,7 @@ function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLe
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
-	const { state, transformers } = createTransformerWithState(useTokenTransformer);
+	const state = { lineInfo: [] };
 	const lang = forcePlainText ? "text" : file.lang ?? getFiletypeFromFileName(file.name);
 	const baseThemeType = typeof theme === "string" ? highlighter.getTheme(theme).type : void 0;
 	const themeStyles = getHighlighterThemeStyles({
@@ -1700,27 +1926,12 @@ function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLe
 		lineIndex: shikiLineNumber - 1 + startingLine,
 		lineNumber: shikiLineNumber + startingLine
 	});
-	const hastConfig = (() => {
-		if (typeof theme === "string") return {
-			lang,
-			theme,
-			transformers,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeMaxLineLength,
-			tokenizeTimeLimit: 0
-		};
-		return {
-			lang,
-			themes: theme,
-			transformers,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeMaxLineLength,
-			tokenizeTimeLimit: 0
-		};
-	})();
-	const highlightedLines = getLineNodes(highlighter.codeToHast(normalizeHighlightLineEndings(isWindowedHighlight ? extractWindowedFileContent(lines ?? linesFromFileContents(file.contents), startingLine, totalLines) : file.contents), hastConfig));
+	const highlightedLines = renderTokenLines(highlighter.codeToTokens(normalizeHighlightLineEndings(isWindowedHighlight ? extractWindowedFileContent(lines ?? linesFromFileContents(file.contents), startingLine, totalLines) : file.contents), getTokenOptions(lang, theme, tokenizeMaxLineLength)).tokens, {
+		state,
+		useTokenTransformer,
+		lazyLineAST: lazyLineAST && !forcePlainText && !useTokenTransformer,
+		cacheHtmlStyles: highlighter.name === "highlights"
+	});
 	const code = isWindowedHighlight ? new Array(startingLine) : highlightedLines;
 	if (isWindowedHighlight) appendItems(code, highlightedLines);
 	return {
@@ -1738,8 +1949,102 @@ function extractWindowedFileContent(lines, startingLine, totalLines) {
 	return lines.slice(startingLine, endLine).join("");
 }
 //#endregion
+//#region src/worker/hastLinesTransport.ts
+const HOLE = 0;
+const TEXT = 1;
+const ELEMENT = 2;
+const OTHER = 3;
+const UNDEFINED = 0;
+const NULL = 1;
+const TRUE = 2;
+const FALSE = 3;
+const STRING = 4;
+const INTEGER = 5;
+const EXTRA = 6;
+function encodeHastLines(lines) {
+	let ops = new Int32Array(Math.max(32, Math.min(65536, lines.length * 32)));
+	let length = 0;
+	const offsets = new Uint32Array(lines.length * 2);
+	const strings = [];
+	const stringIds = /* @__PURE__ */ new Map();
+	const extras = [];
+	let text = "";
+	function push(value) {
+		if (length === ops.length) {
+			const next = new Int32Array(ops.length * 2);
+			next.set(ops);
+			ops = next;
+		}
+		ops[length++] = value;
+	}
+	function intern(value) {
+		let id = stringIds.get(value);
+		if (id == null) {
+			id = strings.length;
+			strings.push(value);
+			stringIds.set(value, id);
+		}
+		return id;
+	}
+	function encodeNode(node) {
+		let keys = 0;
+		for (const _ in node) keys++;
+		if (node?.type === "text" && keys === 2) {
+			push(TEXT);
+			push(node.value.length);
+			text += node.value;
+		} else if (node?.type === "element" && keys === 4 && node.properties != null && Object.getPrototypeOf(node.properties) === Object.prototype) {
+			push(ELEMENT);
+			push(intern(node.tagName));
+			let count = 0;
+			for (const _ in node.properties) count++;
+			push(count);
+			for (const key in node.properties) {
+				const value = node.properties[key];
+				push(intern(key));
+				if (value === void 0) push(UNDEFINED);
+				else if (value === null) push(NULL);
+				else if (value === true) push(TRUE);
+				else if (value === false) push(FALSE);
+				else if (typeof value === "string") {
+					push(STRING);
+					push(intern(value));
+				} else if (typeof value === "number" && (value | 0) === value && !Object.is(value, -0)) {
+					push(INTEGER);
+					push(value);
+				} else {
+					push(EXTRA);
+					push(extras.length);
+					extras.push(value);
+				}
+			}
+			push(node.children.length);
+			for (let index = 0; index < node.children.length; index++) if (index in node.children) encodeNode(node.children[index]);
+			else push(HOLE);
+		} else {
+			push(OTHER);
+			push(extras.length);
+			extras.push(node);
+		}
+	}
+	for (let index = 0; index < lines.length; index++) {
+		offsets[index * 2] = length;
+		offsets[index * 2 + 1] = text.length;
+		if (index in lines) encodeNode(lines[index]);
+		else push(HOLE);
+	}
+	return {
+		ops: ops.subarray(0, length),
+		offsets,
+		strings,
+		text,
+		extras
+	};
+}
+//#endregion
 //#region src/worker/worker.ts
 let highlighter;
+let compactResults = false;
 let renderOptions = {
 	theme: DEFAULT_THEMES,
 	useTokenTransformer: false,
@@ -1749,7 +2054,7 @@ let renderOptions = {
 };
 const EMPTY_REGEXP = /(?:)/;
 self.addEventListener("error", (event) => {
-	console.error("[Shiki Worker] Unhandled error:", event.error);
+	console.error("[Diffs Worker] Unhandled error:", event.error);
 });
 self.addEventListener("message", (event) => {
 	handleMessage(event.data);
@@ -1778,7 +2083,7 @@ async function handleMessage(request) {
 		EMPTY_REGEXP.exec("");
 	}
 }
-async function handleInitialize({ id, renderOptions: options, preferredHighlighter, resolvedThemes, resolvedLanguages, customExtensionsVersion, customExtensionMap }) {
+async function handleInitialize({ id, renderOptions: options, preferredHighlighter, resolvedThemes, resolvedLanguages, customExtensionsVersion, customExtensionMap, resultFormat }) {
 	let highlighter = getHighlighter(preferredHighlighter);
 	if ("then" in highlighter) highlighter = await highlighter;
 	syncCustomExtensionsFromRequest({
@@ -1788,6 +2093,7 @@ async function handleInitialize({ id, renderOptions: options, preferredHighlight
 	attachResolvedThemes(resolvedThemes, highlighter);
 	if (resolvedLanguages != null) attachResolvedLanguages(resolvedLanguages, highlighter);
 	renderOptions = options;
+	compactResults = resultFormat === "hast-ops-v1";
 	postMessage({
 		type: "success",
 		id,
@@ -1820,7 +2126,23 @@ async function handleRenderFile({ id, file, resolvedLanguages, customExtensionsV
 		useTokenTransformer: renderOptions.useTokenTransformer,
 		tokenizeMaxLineLength: renderOptions.tokenizeMaxLineLength
 	};
-	sendFileSuccess(id, renderFileWithHighlighter(file, highlighter, fileOptions), fileOptions);
+	const result = renderFileWithHighlighter(file, highlighter, fileOptions);
+	const code = compactResults ? encodeHastLines(result.code) : void 0;
+	const response = {
+		type: "success",
+		requestType: "file",
+		id,
+		result,
+		options: fileOptions,
+		sentAt: Date.now()
+	};
+	postMessage(code == null ? response : {
+		...response,
+		result: {
+			...result,
+			code
+		}
+	}, code == null ? [] : [code.ops.buffer, code.offsets.buffer]);
 }
 async function handleRenderDiff({ id, diff, resolvedLanguages, customExtensionsVersion, customExtensionMap }) {
 	let highlighter = getHighlighter();
@@ -1830,14 +2152,34 @@ async function handleRenderDiff({ id, diff, resolvedLanguages, customExtensionsV
 		customExtensionMap
 	});
 	if (resolvedLanguages != null) attachResolvedLanguages(resolvedLanguages, highlighter);
-	sendDiffSuccess(id, renderDiffWithHighlighter(diff, highlighter, renderOptions), renderOptions);
+	const result = renderDiffWithHighlighter(diff, highlighter, renderOptions);
+	const code = compactResults ? {
+		deletionLines: encodeHastLines(result.code.deletionLines),
+		additionLines: encodeHastLines(result.code.additionLines)
+	} : void 0;
+	const response = {
+		type: "success",
+		requestType: "diff",
+		id,
+		result,
+		options: renderOptions,
+		sentAt: Date.now()
+	};
+	postMessage(code == null ? response : {
+		...response,
+		result: {
+			...result,
+			code
+		}
+	}, code == null ? [] : [
+		code.deletionLines.ops.buffer,
+		code.deletionLines.offsets.buffer,
+		code.additionLines.ops.buffer,
+		code.additionLines.offsets.buffer
+	]);
 }
-function getHighlighter(preferredHighlighter = "shiki-js") {
-	highlighter ??= createHighlighterCore({
-		themes: [],
-		langs: [],
-		engine: preferredHighlighter === "shiki-wasm" ? createOnigurumaEngine(import("shiki/wasm")) : createJavaScriptRegexEngine()
-	});
+function getHighlighter(preferredHighlighter) {
+	highlighter ??= createHighlighter(preferredHighlighter);
 	return highlighter;
 }
 function syncCustomExtensionsFromRequest({ customExtensionsVersion, customExtensionMap }) {
@@ -1845,26 +2187,6 @@ function syncCustomExtensionsFromRequest({ customExtensionsVersion, customExtens
 	if (customExtensionsVersion == null || customExtensionMap == null) throw new Error("Worker request must include both customExtensionsVersion and customExtensionMap");
 	replaceCustomExtensions(customExtensionsVersion, customExtensionMap);
 }
-function sendFileSuccess(id, result, options) {
-	postMessage({
-		type: "success",
-		requestType: "file",
-		id,
-		result,
-		options,
-		sentAt: Date.now()
-	});
-}
-function sendDiffSuccess(id, result, options) {
-	postMessage({
-		type: "success",
-		requestType: "diff",
-		id,
-		result,
-		options,
-		sentAt: Date.now()
-	});
-}
 function sendError(id, error) {
 	const response = {
 		type: "error",
@@ -1875,5 +2197,6 @@ function sendError(id, error) {
 	postMessage(response);
 }
 //#endregion
+export { acquireHighlighterType as a, RegisteredCustomLanguages as c, HighlighterDisposedError as i, ResolvedLanguages as l, formatCSSVariablePrefix as n, releaseHighlighterType as o, appendItems as r, attachResolvedLanguages as s, renderTokenLines as t, ResolvingLanguages as u };
 
 //# sourceMappingURL=worker.js.map