@pierre/diffs 1.5.2 → 1.6.0

package/dist/utils/renderDiffWithHighlighter.js

@@ -1,17 +1,17 @@
 import "../constants.js";
 import { getFiletypeFromFileName } from "./getFiletypeFromFileName.js";
 import { appendItems } from "./appendItems.js";
-import { createTransformerWithState } from "./createTransformerWithState.js";
-import { formatCSSVariablePrefix } from "./formatCSSVariablePrefix.js";
 import { getHighlighterThemeStyles } from "./getHighlighterThemeStyles.js";
-import { getLineNodes } from "./getLineNodes.js";
+import { getTokenOptions } from "./getTokenOptions.js";
+import { setDeferredArrayItem } from "./setDeferredArrayItem.js";
+import { renderTokenLines } from "./renderTokenLines.js";
 import { iterateOverDiff } from "./iterateOverDiff.js";
 import { cleanLastNewline } from "./cleanLastNewline.js";
 import { createDiffSpanDecoration, pushOrJoinSpan } from "./parseDiffDecorations.js";
 import { diffChars, diffWordsWithSpace } from "diff";
 //#region src/utils/renderDiffWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS = { forcePlainText: false };
-function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText, startingLine, totalLines, expandedHunks, collapsedContextThreshold = 1 } = DEFAULT_PLAIN_TEXT_OPTIONS) {
+function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText, lazyLineAST = false, startingLine, totalLines, expandedHunks, collapsedContextThreshold = 1 } = DEFAULT_PLAIN_TEXT_OPTIONS) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -20,6 +20,7 @@ function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText,
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
+	lazyLineAST &&= !forcePlainText && !options.useTokenTransformer;
 	const baseThemeType = typeof options.theme === "string" ? highlighter.getTheme(options.theme).type : void 0;
 	const themeStyles = getHighlighterThemeStyles({
 		theme: options.theme,
@@ -114,17 +115,27 @@ function renderDiffWithHighlighter(diff, highlighter, options, { forcePlainText,
 			additionDecorations: bucket.additionDecorations,
 			highlighter,
 			options,
-			languageOverride: forcePlainText ? "text" : diff.lang
+			languageOverride: forcePlainText ? "text" : diff.lang,
+			lazyLineAST
 		});
 		if (shouldGroupAll) {
 			code.deletionLines = deletionLines;
 			code.additionLines = additionLines;
 			continue;
 		}
-		if (bucket.deletionSegments.length > 0) for (const seg of bucket.deletionSegments) for (let i = 0; i < seg.count; i++) code.deletionLines[seg.targetIndex + i] = deletionLines[seg.originalOffset + i];
-		else appendItems(code.deletionLines, deletionLines);
-		if (bucket.additionSegments.length > 0) for (const seg of bucket.additionSegments) for (let i = 0; i < seg.count; i++) code.additionLines[seg.targetIndex + i] = additionLines[seg.originalOffset + i];
-		else appendItems(code.additionLines, additionLines);
+		for (const [target, lines, segments] of [[
+			code.deletionLines,
+			deletionLines,
+			bucket.deletionSegments
+		], [
+			code.additionLines,
+			additionLines,
+			bucket.additionSegments
+		]]) if (segments.length > 0) for (const segment of segments) for (let index = 0; index < segment.count; index++) target[segment.targetIndex + index] = lines[segment.originalOffset + index];
+		else if (lazyLineAST) {
+			const offset = target.length;
+			for (let index = 0; index < lines.length; index++) setDeferredArrayItem(target, offset + index, () => lines[index]);
+		} else appendItems(target, lines);
 	}
 	return {
 		code,
@@ -256,46 +267,20 @@ function createBucket() {
 		additionSegments: []
 	};
 }
-function renderTwoFiles({ deletionFile, additionFile, deletionInfo, additionInfo, highlighter, deletionDecorations, additionDecorations, languageOverride, options: { theme: themeOrThemes, ...options } }) {
-	const deletionLang = languageOverride ?? getFiletypeFromFileName(deletionFile.name);
-	const additionLang = languageOverride ?? getFiletypeFromFileName(additionFile.name);
-	const { state, transformers } = createTransformerWithState(options.useTokenTransformer);
-	const hastConfig = (() => {
-		return typeof themeOrThemes === "string" ? {
-			...options,
-			lang: "text",
-			theme: themeOrThemes,
-			transformers,
-			decorations: void 0,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		} : {
-			...options,
-			lang: "text",
-			themes: themeOrThemes,
-			transformers,
-			decorations: void 0,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		};
-	})();
+function renderTwoFiles({ lazyLineAST, deletionFile, additionFile, deletionInfo, additionInfo, highlighter, deletionDecorations, additionDecorations, languageOverride, options: { theme, useTokenTransformer, tokenizeMaxLineLength } }) {
+	const renderFile = (file, lineInfo, decorations) => {
+		if (file.contents === "") return [];
+		return renderTokenLines(highlighter.codeToTokens(cleanLastNewline(file.contents), getTokenOptions(languageOverride ?? getFiletypeFromFileName(file.name), theme, tokenizeMaxLineLength)).tokens, {
+			state: { lineInfo },
+			useTokenTransformer,
+			decorations,
+			lazyLineAST,
+			cacheHtmlStyles: highlighter.name === "highlights"
+		});
+	};
 	return {
-		deletionLines: (() => {
-			if (deletionFile.contents === "") return [];
-			hastConfig.lang = deletionLang;
-			state.lineInfo = deletionInfo;
-			hastConfig.decorations = deletionDecorations;
-			return getLineNodes(highlighter.codeToHast(cleanLastNewline(deletionFile.contents), hastConfig));
-		})(),
-		additionLines: (() => {
-			if (additionFile.contents === "") return [];
-			hastConfig.lang = additionLang;
-			hastConfig.decorations = additionDecorations;
-			state.lineInfo = additionInfo;
-			return getLineNodes(highlighter.codeToHast(cleanLastNewline(additionFile.contents), hastConfig));
-		})()
+		deletionLines: renderFile(deletionFile, deletionInfo, deletionDecorations),
+		additionLines: renderFile(additionFile, additionInfo, additionDecorations)
 	};
 }
 //#endregion

package/dist/utils/renderDiffWithHighlighter.js.map

@@ -1 +1 @@
-{"version":3,"file":"renderDiffWithHighlighter.js","names":[],"sources":["../../src/utils/renderDiffWithHighlighter.ts"],"sourcesContent":["import { type ChangeObject, diffChars, diffWordsWithSpace } from 'diff';\n\nimport { DEFAULT_COLLAPSED_CONTEXT_THRESHOLD } from '../constants';\nimport type {\n  CodeToHastOptions,\n  DecorationItem,\n  DiffsHighlighter,\n  DiffsThemeNames,\n  FileContents,\n  FileDiffMetadata,\n  ForceDiffPlainTextOptions,\n  LineDiffTypes,\n  LineInfo,\n  RenderDiffFilesResult,\n  RenderDiffOptions,\n  SupportedLanguages,\n  ThemedDiffResult,\n} from '../types';\nimport { appendItems } from './appendItems';\nimport { cleanLastNewline } from './cleanLastNewline';\nimport { createTransformerWithState } from './createTransformerWithState';\nimport { formatCSSVariablePrefix } from './formatCSSVariablePrefix';\nimport { getFiletypeFromFileName } from './getFiletypeFromFileName';\nimport { getHighlighterThemeStyles } from './getHighlighterThemeStyles';\nimport { getLineNodes } from './getLineNodes';\nimport { iterateOverDiff } from './iterateOverDiff';\nimport {\n  createDiffSpanDecoration,\n  pushOrJoinSpan,\n} from './parseDiffDecorations';\n\nconst DEFAULT_PLAIN_TEXT_OPTIONS: ForceDiffPlainTextOptions = {\n  forcePlainText: false,\n};\n\nexport function renderDiffWithHighlighter(\n  diff: FileDiffMetadata,\n  highlighter: DiffsHighlighter,\n  options: RenderDiffOptions,\n  {\n    forcePlainText,\n    startingLine,\n    totalLines,\n    expandedHunks,\n    collapsedContextThreshold = DEFAULT_COLLAPSED_CONTEXT_THRESHOLD,\n  }: ForceDiffPlainTextOptions = DEFAULT_PLAIN_TEXT_OPTIONS\n): ThemedDiffResult {\n  if (forcePlainText) {\n    startingLine ??= 0;\n    totalLines ??= Infinity;\n  } else {\n    // If we aren't forcing plain text, then we intentionally do not support\n    // ranges for highlighting as that could break the syntax highlighting, we\n    // we override any values that may have been passed in.  Maybe one day we\n    // warn about this?\n    startingLine = 0;\n    totalLines = Infinity;\n  }\n  const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;\n  const baseThemeType =\n    typeof options.theme === 'string'\n      ? highlighter.getTheme(options.theme).type\n      : undefined;\n  const themeStyles = getHighlighterThemeStyles({\n    theme: options.theme,\n    highlighter,\n  });\n\n  // If we have a large file and we are rendering the WHOLE plain diff ast,\n  // then we should remove the lineDiffType to make sure things render quickly.\n  // For highlighted ASTs or windowed highlights, we should just inherit the\n  // setting\n  const lineDiffType =\n    forcePlainText &&\n    !isWindowedHighlight &&\n    (diff.unifiedLineCount > 1000 || diff.splitLineCount > 1000)\n      ? 'none'\n      : options.lineDiffType;\n\n  const code: RenderDiffFilesResult = {\n    deletionLines: [],\n    additionLines: [],\n  };\n\n  const { maxLineDiffLength } = options;\n  const shouldGroupAll = !forcePlainText && !diff.isPartial;\n  const expandedHunksForIteration = forcePlainText ? expandedHunks : undefined;\n  const buckets = new Map<number, RenderBucket>();\n  function getBucketForHunk(hunkIndex: number) {\n    const index = shouldGroupAll ? 0 : hunkIndex;\n    const bucket = buckets.get(index) ?? createBucket();\n    buckets.set(index, bucket);\n    return bucket;\n  }\n\n  function appendContent(\n    lineContent: string,\n    lineIndex: number,\n    segments: HighlightSegment[],\n    contentWrapper: FakeArrayType\n  ) {\n    if (isWindowedHighlight) {\n      let segment = segments.at(-1);\n      if (\n        segment == null ||\n        segment.targetIndex + segment.count !== lineIndex\n      ) {\n        segment = {\n          targetIndex: lineIndex,\n          originalOffset: contentWrapper.length,\n          count: 0,\n        };\n        segments.push(segment);\n      }\n      segment.count++;\n    }\n    contentWrapper.push(lineContent);\n  }\n\n  iterateOverDiff({\n    diff,\n    diffStyle: 'both',\n    startingLine,\n    totalLines,\n    expandedHunks: isWindowedHighlight ? expandedHunksForIteration : true,\n    collapsedContextThreshold,\n    callback: ({ hunkIndex, additionLine, deletionLine, type }) => {\n      const bucket = getBucketForHunk(hunkIndex);\n      const splitLineIndex =\n        additionLine != null\n          ? additionLine.splitLineIndex\n          : deletionLine.splitLineIndex;\n\n      if (type === 'change' && additionLine != null && deletionLine != null) {\n        computeLineDiffDecorations({\n          additionLine: diff.additionLines[additionLine.lineIndex],\n          deletionLine: diff.deletionLines[deletionLine.lineIndex],\n          deletionLineIndex: bucket.deletionContent.length,\n          additionLineIndex: bucket.additionContent.length,\n          deletionDecorations: bucket.deletionDecorations,\n          additionDecorations: bucket.additionDecorations,\n          lineDiffType,\n          maxLineDiffLength,\n        });\n      }\n\n      if (deletionLine != null) {\n        appendContent(\n          diff.deletionLines[deletionLine.lineIndex],\n          deletionLine.lineIndex,\n          bucket.deletionSegments,\n          bucket.deletionContent\n        );\n        bucket.deletionInfo.push({\n          type: type === 'change' ? 'change-deletion' : type,\n          lineNumber: deletionLine.lineNumber,\n          altLineNumber:\n            type === 'change'\n              ? undefined\n              : (additionLine.lineNumber ?? undefined),\n          lineIndex: `${deletionLine.unifiedLineIndex},${splitLineIndex}`,\n        });\n      }\n\n      if (additionLine != null) {\n        appendContent(\n          diff.additionLines[additionLine.lineIndex],\n          additionLine.lineIndex,\n          bucket.additionSegments,\n          bucket.additionContent\n        );\n        bucket.additionInfo.push({\n          type: type === 'change' ? 'change-addition' : type,\n          lineNumber: additionLine.lineNumber,\n          altLineNumber:\n            type === 'change'\n              ? undefined\n              : (deletionLine.lineNumber ?? undefined),\n          lineIndex: `${additionLine.unifiedLineIndex},${splitLineIndex}`,\n        });\n      }\n    },\n  });\n\n  for (const bucket of buckets.values()) {\n    if (\n      bucket.deletionContent.length === 0 &&\n      bucket.additionContent.length === 0\n    ) {\n      continue;\n    }\n\n    const deletionFile = {\n      name: diff.prevName ?? diff.name,\n      contents: bucket.deletionContent.value,\n    };\n    const additionFile = {\n      name: diff.name,\n      contents: bucket.additionContent.value,\n    };\n    const { deletionLines, additionLines } = renderTwoFiles({\n      deletionFile,\n      deletionInfo: bucket.deletionInfo,\n      deletionDecorations: bucket.deletionDecorations,\n\n      additionFile,\n      additionInfo: bucket.additionInfo,\n      additionDecorations: bucket.additionDecorations,\n\n      highlighter,\n      options,\n      languageOverride: forcePlainText ? 'text' : diff.lang,\n    });\n\n    if (shouldGroupAll) {\n      code.deletionLines = deletionLines;\n      code.additionLines = additionLines;\n      continue;\n    }\n\n    if (bucket.deletionSegments.length > 0) {\n      for (const seg of bucket.deletionSegments) {\n        for (let i = 0; i < seg.count; i++) {\n          code.deletionLines[seg.targetIndex + i] =\n            deletionLines[seg.originalOffset + i];\n        }\n      }\n    } else {\n      appendItems(code.deletionLines, deletionLines);\n    }\n    if (bucket.additionSegments.length > 0) {\n      for (const seg of bucket.additionSegments) {\n        for (let i = 0; i < seg.count; i++) {\n          code.additionLines[seg.targetIndex + i] =\n            additionLines[seg.originalOffset + i];\n        }\n      }\n    } else {\n      appendItems(code.additionLines, additionLines);\n    }\n  }\n\n  return { code, themeStyles, baseThemeType };\n}\n\n// The line indexes and decoration arrays a line diff writes its highlights to\ninterface LineDiffDecorationTargets {\n  deletionLineIndex: number;\n  additionLineIndex: number;\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n}\n\ninterface ProcessLineDiffProps extends LineDiffDecorationTargets {\n  deletionLine: string | undefined;\n  additionLine: string | undefined;\n  lineDiffType: LineDiffTypes;\n  maxLineDiffLength: number;\n}\n\n// Push one decoration per side covering the first through last changed\n// token, so each line only has one highlight. Offsets come straight from the\n// diff items, skipping the span arrays the other word modes build.\nfunction pushWordLineDecorations(\n  lineDiff: ChangeObject<string>[],\n  {\n    deletionLineIndex,\n    additionLineIndex,\n    deletionDecorations,\n    additionDecorations,\n  }: LineDiffDecorationTargets\n): void {\n  let deletionOffset = 0;\n  let additionOffset = 0;\n  let deletionStart = -1;\n  let deletionEnd = 0;\n  let additionStart = -1;\n  let additionEnd = 0;\n  for (const { value, added, removed } of lineDiff) {\n    if (removed) {\n      if (deletionStart === -1) deletionStart = deletionOffset;\n      deletionOffset += value.length;\n      deletionEnd = deletionOffset;\n    } else if (added) {\n      if (additionStart === -1) additionStart = additionOffset;\n      additionOffset += value.length;\n      additionEnd = additionOffset;\n    } else {\n      deletionOffset += value.length;\n      additionOffset += value.length;\n    }\n  }\n  if (deletionStart !== -1) {\n    deletionDecorations.push(\n      createDiffSpanDecoration({\n        line: deletionLineIndex,\n        spanStart: deletionStart,\n        spanLength: deletionEnd - deletionStart,\n      })\n    );\n  }\n  if (additionStart !== -1) {\n    additionDecorations.push(\n      createDiffSpanDecoration({\n        line: additionLineIndex,\n        spanStart: additionStart,\n        spanLength: additionEnd - additionStart,\n      })\n    );\n  }\n}\n\nfunction computeLineDiffDecorations({\n  deletionLine,\n  additionLine,\n  deletionLineIndex,\n  additionLineIndex,\n  deletionDecorations,\n  additionDecorations,\n  lineDiffType,\n  maxLineDiffLength,\n}: ProcessLineDiffProps) {\n  if (deletionLine == null || additionLine == null || lineDiffType === 'none') {\n    return;\n  }\n  deletionLine = cleanLastNewline(deletionLine);\n  additionLine = cleanLastNewline(additionLine);\n  // If we have really long lines, we probably shouldn't compute diffs on them.\n  if (\n    deletionLine.length > maxLineDiffLength ||\n    additionLine.length > maxLineDiffLength\n  ) {\n    return;\n  }\n  // NOTE(amadeus): Because we visually trim trailing newlines when rendering,\n  // we also gotta make sure the diff parsing doesn't include the newline\n  // character that could be there...\n  const lineDiff =\n    lineDiffType === 'char'\n      ? diffChars(deletionLine, additionLine)\n      : diffWordsWithSpace(deletionLine, additionLine);\n  if (lineDiffType === 'word-line') {\n    pushWordLineDecorations(lineDiff, {\n      deletionLineIndex,\n      additionLineIndex,\n      deletionDecorations,\n      additionDecorations,\n    });\n    return;\n  }\n  const deletionSpans: [0 | 1, string][] = [];\n  const additionSpans: [0 | 1, string][] = [];\n  const enableJoin = lineDiffType === 'word-alt';\n  const lastItem = lineDiff.at(-1);\n  for (const item of lineDiff) {\n    const isLastItem = item === lastItem;\n    if (!item.added && !item.removed) {\n      pushOrJoinSpan({\n        item,\n        arr: deletionSpans,\n        enableJoin,\n        isNeutral: true,\n        isLastItem,\n      });\n      pushOrJoinSpan({\n        item,\n        arr: additionSpans,\n        enableJoin,\n        isNeutral: true,\n        isLastItem,\n      });\n    } else if (item.removed) {\n      pushOrJoinSpan({ item, arr: deletionSpans, enableJoin, isLastItem });\n    } else {\n      pushOrJoinSpan({ item, arr: additionSpans, enableJoin, isLastItem });\n    }\n  }\n  let spanIndex = 0;\n  for (const span of deletionSpans) {\n    if (span[0] === 1) {\n      deletionDecorations.push(\n        createDiffSpanDecoration({\n          line: deletionLineIndex,\n          spanStart: spanIndex,\n          spanLength: span[1].length,\n        })\n      );\n    }\n    spanIndex += span[1].length;\n  }\n  spanIndex = 0;\n  for (const span of additionSpans) {\n    if (span[0] === 1) {\n      additionDecorations.push(\n        createDiffSpanDecoration({\n          line: additionLineIndex,\n          spanStart: spanIndex,\n          spanLength: span[1].length,\n        })\n      );\n    }\n    spanIndex += span[1].length;\n  }\n}\n\ninterface HighlightSegment {\n  // The where the highlighted region starts\n  originalOffset: number;\n  // Where to place the highlighted line in RenderDiffFilesResult\n  targetIndex: number;\n  // Number of highlighted lines\n  count: number;\n}\n\ninterface FakeArrayType {\n  push(value: string): void;\n  value: string;\n  length: number;\n}\n\ninterface RenderBucket {\n  deletionContent: FakeArrayType;\n  additionContent: FakeArrayType;\n  deletionInfo: (LineInfo | undefined)[];\n  additionInfo: (LineInfo | undefined)[];\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n  deletionSegments: HighlightSegment[];\n  additionSegments: HighlightSegment[];\n}\n\nfunction createBucket(): RenderBucket {\n  return {\n    deletionContent: {\n      push(value: string) {\n        this.value += value;\n        this.length++;\n      },\n      value: '',\n      length: 0,\n    },\n    additionContent: {\n      push(value: string) {\n        this.value += value;\n        this.length++;\n      },\n      value: '',\n      length: 0,\n    },\n    deletionInfo: [],\n    additionInfo: [],\n    deletionDecorations: [],\n    additionDecorations: [],\n    deletionSegments: [],\n    additionSegments: [],\n  };\n}\n\ninterface RenderTwoFilesProps {\n  deletionFile: FileContents;\n  additionFile: FileContents;\n  deletionInfo: (LineInfo | undefined)[];\n  additionInfo: (LineInfo | undefined)[];\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n  options: RenderDiffOptions;\n  highlighter: DiffsHighlighter;\n  languageOverride: SupportedLanguages | undefined;\n}\n\nfunction renderTwoFiles({\n  deletionFile,\n  additionFile,\n  deletionInfo,\n  additionInfo,\n  highlighter,\n  deletionDecorations,\n  additionDecorations,\n  languageOverride,\n  options: { theme: themeOrThemes, ...options },\n}: RenderTwoFilesProps): RenderDiffFilesResult {\n  const deletionLang =\n    languageOverride ?? getFiletypeFromFileName(deletionFile.name);\n  const additionLang =\n    languageOverride ?? getFiletypeFromFileName(additionFile.name);\n  const { state, transformers } = createTransformerWithState(\n    options.useTokenTransformer\n  );\n  // tokenizeTimeLimit: 0 — never trade silently-wrong token colors for\n  // latency; see renderFileWithHighlighter for the full rationale.\n  const hastConfig: CodeToHastOptions<DiffsThemeNames> = (() => {\n    return typeof themeOrThemes === 'string'\n      ? {\n          ...options,\n          // language will be overwritten for each highlight\n          lang: 'text',\n          theme: themeOrThemes,\n          transformers,\n          decorations: undefined,\n          defaultColor: false,\n          cssVariablePrefix: formatCSSVariablePrefix('token'),\n          tokenizeTimeLimit: 0,\n        }\n      : {\n          ...options,\n          // language will be overwritten for each highlight\n          lang: 'text',\n          themes: themeOrThemes,\n          transformers,\n          decorations: undefined,\n          defaultColor: false,\n          cssVariablePrefix: formatCSSVariablePrefix('token'),\n          tokenizeTimeLimit: 0,\n        };\n  })();\n\n  const deletionLines = (() => {\n    if (deletionFile.contents === '') {\n      return [];\n    }\n    hastConfig.lang = deletionLang;\n    state.lineInfo = deletionInfo;\n    hastConfig.decorations = deletionDecorations;\n    return getLineNodes(\n      highlighter.codeToHast(\n        cleanLastNewline(deletionFile.contents),\n        hastConfig\n      )\n    );\n  })();\n  const additionLines = (() => {\n    if (additionFile.contents === '') {\n      return [];\n    }\n    hastConfig.lang = additionLang;\n    hastConfig.decorations = additionDecorations;\n    state.lineInfo = additionInfo;\n    return getLineNodes(\n      highlighter.codeToHast(\n        cleanLastNewline(additionFile.contents),\n        hastConfig\n      )\n    );\n  })();\n\n  return { deletionLines, additionLines };\n}\n"],"mappings":";;;;;;;;;;;;AA+BA,MAAM,6BAAwD,EAC5D,gBAAgB,MAClB;AAEA,SAAgB,0BACd,MACA,aACA,SACA,EACE,gBACA,cACA,YACA,eACA,4BAAA,MAC6B,4BACb;CAClB,IAAI,gBAAgB;EAClB,iBAAiB;EACjB,eAAe;CACjB,OAAO;EAKL,eAAe;EACf,aAAa;CACf;CACA,MAAM,sBAAsB,eAAe,KAAK,aAAa;CAC7D,MAAM,gBACJ,OAAO,QAAQ,UAAU,WACrB,YAAY,SAAS,QAAQ,KAAK,CAAC,CAAC,OACpC,KAAA;CACN,MAAM,cAAc,0BAA0B;EAC5C,OAAO,QAAQ;EACf;CACF,CAAC;CAMD,MAAM,eACJ,kBACA,CAAC,wBACA,KAAK,mBAAmB,OAAQ,KAAK,iBAAiB,OACnD,SACA,QAAQ;CAEd,MAAM,OAA8B;EAClC,eAAe,CAAC;EAChB,eAAe,CAAC;CAClB;CAEA,MAAM,EAAE,sBAAsB;CAC9B,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,KAAK;CAChD,MAAM,4BAA4B,iBAAiB,gBAAgB,KAAA;CACnE,MAAM,0BAAU,IAAI,IAA0B;CAC9C,SAAS,iBAAiB,WAAmB;EAC3C,MAAM,QAAQ,iBAAiB,IAAI;EACnC,MAAM,SAAS,QAAQ,IAAI,KAAK,KAAK,aAAa;EAClD,QAAQ,IAAI,OAAO,MAAM;EACzB,OAAO;CACT;CAEA,SAAS,cACP,aACA,WACA,UACA,gBACA;EACA,IAAI,qBAAqB;GACvB,IAAI,UAAU,SAAS,GAAG,EAAE;GAC5B,IACE,WAAW,QACX,QAAQ,cAAc,QAAQ,UAAU,WACxC;IACA,UAAU;KACR,aAAa;KACb,gBAAgB,eAAe;KAC/B,OAAO;IACT;IACA,SAAS,KAAK,OAAO;GACvB;GACA,QAAQ;EACV;EACA,eAAe,KAAK,WAAW;CACjC;CAEA,gBAAgB;EACd;EACA,WAAW;EACX;EACA;EACA,eAAe,sBAAsB,4BAA4B;EACjE;EACA,WAAW,EAAE,WAAW,cAAc,cAAc,WAAW;GAC7D,MAAM,SAAS,iBAAiB,SAAS;GACzC,MAAM,iBACJ,gBAAgB,OACZ,aAAa,iBACb,aAAa;GAEnB,IAAI,SAAS,YAAY,gBAAgB,QAAQ,gBAAgB,MAC/D,2BAA2B;IACzB,cAAc,KAAK,cAAc,aAAa;IAC9C,cAAc,KAAK,cAAc,aAAa;IAC9C,mBAAmB,OAAO,gBAAgB;IAC1C,mBAAmB,OAAO,gBAAgB;IAC1C,qBAAqB,OAAO;IAC5B,qBAAqB,OAAO;IAC5B;IACA;GACF,CAAC;GAGH,IAAI,gBAAgB,MAAM;IACxB,cACE,KAAK,cAAc,aAAa,YAChC,aAAa,WACb,OAAO,kBACP,OAAO,eACT;IACA,OAAO,aAAa,KAAK;KACvB,MAAM,SAAS,WAAW,oBAAoB;KAC9C,YAAY,aAAa;KACzB,eACE,SAAS,WACL,KAAA,IACC,aAAa,cAAc,KAAA;KAClC,WAAW,GAAG,aAAa,iBAAiB,GAAG;IACjD,CAAC;GACH;GAEA,IAAI,gBAAgB,MAAM;IACxB,cACE,KAAK,cAAc,aAAa,YAChC,aAAa,WACb,OAAO,kBACP,OAAO,eACT;IACA,OAAO,aAAa,KAAK;KACvB,MAAM,SAAS,WAAW,oBAAoB;KAC9C,YAAY,aAAa;KACzB,eACE,SAAS,WACL,KAAA,IACC,aAAa,cAAc,KAAA;KAClC,WAAW,GAAG,aAAa,iBAAiB,GAAG;IACjD,CAAC;GACH;EACF;CACF,CAAC;CAED,KAAK,MAAM,UAAU,QAAQ,OAAO,GAAG;EACrC,IACE,OAAO,gBAAgB,WAAW,KAClC,OAAO,gBAAgB,WAAW,GAElC;EAGF,MAAM,eAAe;GACnB,MAAM,KAAK,YAAY,KAAK;GAC5B,UAAU,OAAO,gBAAgB;EACnC;EACA,MAAM,eAAe;GACnB,MAAM,KAAK;GACX,UAAU,OAAO,gBAAgB;EACnC;EACA,MAAM,EAAE,eAAe,kBAAkB,eAAe;GACtD;GACA,cAAc,OAAO;GACrB,qBAAqB,OAAO;GAE5B;GACA,cAAc,OAAO;GACrB,qBAAqB,OAAO;GAE5B;GACA;GACA,kBAAkB,iBAAiB,SAAS,KAAK;EACnD,CAAC;EAED,IAAI,gBAAgB;GAClB,KAAK,gBAAgB;GACrB,KAAK,gBAAgB;GACrB;EACF;EAEA,IAAI,OAAO,iBAAiB,SAAS,GACnC,KAAK,MAAM,OAAO,OAAO,kBACvB,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,OAAO,KAC7B,KAAK,cAAc,IAAI,cAAc,KACnC,cAAc,IAAI,iBAAiB;OAIzC,YAAY,KAAK,eAAe,aAAa;EAE/C,IAAI,OAAO,iBAAiB,SAAS,GACnC,KAAK,MAAM,OAAO,OAAO,kBACvB,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,OAAO,KAC7B,KAAK,cAAc,IAAI,cAAc,KACnC,cAAc,IAAI,iBAAiB;OAIzC,YAAY,KAAK,eAAe,aAAa;CAEjD;CAEA,OAAO;EAAE;EAAM;EAAa;CAAc;AAC5C;AAoBA,SAAS,wBACP,UACA,EACE,mBACA,mBACA,qBACA,uBAEI;CACN,IAAI,iBAAiB;CACrB,IAAI,iBAAiB;CACrB,IAAI,gBAAgB;CACpB,IAAI,cAAc;CAClB,IAAI,gBAAgB;CACpB,IAAI,cAAc;CAClB,KAAK,MAAM,EAAE,OAAO,OAAO,aAAa,UACtC,IAAI,SAAS;EACX,IAAI,kBAAkB,IAAI,gBAAgB;EAC1C,kBAAkB,MAAM;EACxB,cAAc;CAChB,OAAO,IAAI,OAAO;EAChB,IAAI,kBAAkB,IAAI,gBAAgB;EAC1C,kBAAkB,MAAM;EACxB,cAAc;CAChB,OAAO;EACL,kBAAkB,MAAM;EACxB,kBAAkB,MAAM;CAC1B;CAEF,IAAI,kBAAkB,IACpB,oBAAoB,KAClB,yBAAyB;EACvB,MAAM;EACN,WAAW;EACX,YAAY,cAAc;CAC5B,CAAC,CACH;CAEF,IAAI,kBAAkB,IACpB,oBAAoB,KAClB,yBAAyB;EACvB,MAAM;EACN,WAAW;EACX,YAAY,cAAc;CAC5B,CAAC,CACH;AAEJ;AAEA,SAAS,2BAA2B,EAClC,cACA,cACA,mBACA,mBACA,qBACA,qBACA,cACA,qBACuB;CACvB,IAAI,gBAAgB,QAAQ,gBAAgB,QAAQ,iBAAiB,QACnE;CAEF,eAAe,iBAAiB,YAAY;CAC5C,eAAe,iBAAiB,YAAY;CAE5C,IACE,aAAa,SAAS,qBACtB,aAAa,SAAS,mBAEtB;CAKF,MAAM,WACJ,iBAAiB,SACb,UAAU,cAAc,YAAY,IACpC,mBAAmB,cAAc,YAAY;CACnD,IAAI,iBAAiB,aAAa;EAChC,wBAAwB,UAAU;GAChC;GACA;GACA;GACA;EACF,CAAC;EACD;CACF;CACA,MAAM,gBAAmC,CAAC;CAC1C,MAAM,gBAAmC,CAAC;CAC1C,MAAM,aAAa,iBAAiB;CACpC,MAAM,WAAW,SAAS,GAAG,EAAE;CAC/B,KAAK,MAAM,QAAQ,UAAU;EAC3B,MAAM,aAAa,SAAS;EAC5B,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,SAAS;GAChC,eAAe;IACb;IACA,KAAK;IACL;IACA,WAAW;IACX;GACF,CAAC;GACD,eAAe;IACb;IACA,KAAK;IACL;IACA,WAAW;IACX;GACF,CAAC;EACH,OAAO,IAAI,KAAK,SACd,eAAe;GAAE;GAAM,KAAK;GAAe;GAAY;EAAW,CAAC;OAEnE,eAAe;GAAE;GAAM,KAAK;GAAe;GAAY;EAAW,CAAC;CAEvE;CACA,IAAI,YAAY;CAChB,KAAK,MAAM,QAAQ,eAAe;EAChC,IAAI,KAAK,OAAO,GACd,oBAAoB,KAClB,yBAAyB;GACvB,MAAM;GACN,WAAW;GACX,YAAY,KAAK,EAAE,CAAC;EACtB,CAAC,CACH;EAEF,aAAa,KAAK,EAAE,CAAC;CACvB;CACA,YAAY;CACZ,KAAK,MAAM,QAAQ,eAAe;EAChC,IAAI,KAAK,OAAO,GACd,oBAAoB,KAClB,yBAAyB;GACvB,MAAM;GACN,WAAW;GACX,YAAY,KAAK,EAAE,CAAC;EACtB,CAAC,CACH;EAEF,aAAa,KAAK,EAAE,CAAC;CACvB;AACF;AA4BA,SAAS,eAA6B;CACpC,OAAO;EACL,iBAAiB;GACf,KAAK,OAAe;IAClB,KAAK,SAAS;IACd,KAAK;GACP;GACA,OAAO;GACP,QAAQ;EACV;EACA,iBAAiB;GACf,KAAK,OAAe;IAClB,KAAK,SAAS;IACd,KAAK;GACP;GACA,OAAO;GACP,QAAQ;EACV;EACA,cAAc,CAAC;EACf,cAAc,CAAC;EACf,qBAAqB,CAAC;EACtB,qBAAqB,CAAC;EACtB,kBAAkB,CAAC;EACnB,kBAAkB,CAAC;CACrB;AACF;AAcA,SAAS,eAAe,EACtB,cACA,cACA,cACA,cACA,aACA,qBACA,qBACA,kBACA,SAAS,EAAE,OAAO,eAAe,GAAG,aACS;CAC7C,MAAM,eACJ,oBAAoB,wBAAwB,aAAa,IAAI;CAC/D,MAAM,eACJ,oBAAoB,wBAAwB,aAAa,IAAI;CAC/D,MAAM,EAAE,OAAO,iBAAiB,2BAC9B,QAAQ,mBACV;CAGA,MAAM,oBAAwD;EAC5D,OAAO,OAAO,kBAAkB,WAC5B;GACE,GAAG;GAEH,MAAM;GACN,OAAO;GACP;GACA,aAAa,KAAA;GACb,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD,mBAAmB;EACrB,IACA;GACE,GAAG;GAEH,MAAM;GACN,QAAQ;GACR;GACA,aAAa,KAAA;GACb,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD,mBAAmB;EACrB;CACN,EAAA,CAAG;CA+BH,OAAO;EAAE,sBA7BoB;GAC3B,IAAI,aAAa,aAAa,IAC5B,OAAO,CAAC;GAEV,WAAW,OAAO;GAClB,MAAM,WAAW;GACjB,WAAW,cAAc;GACzB,OAAO,aACL,YAAY,WACV,iBAAiB,aAAa,QAAQ,GACtC,UACF,CACF;EACF,EAAA,CAgBqB;EAAG,sBAfK;GAC3B,IAAI,aAAa,aAAa,IAC5B,OAAO,CAAC;GAEV,WAAW,OAAO;GAClB,WAAW,cAAc;GACzB,MAAM,WAAW;GACjB,OAAO,aACL,YAAY,WACV,iBAAiB,aAAa,QAAQ,GACtC,UACF,CACF;EACF,EAAA,CAEoC;CAAE;AACxC"}
+{"version":3,"file":"renderDiffWithHighlighter.js","names":[],"sources":["../../src/utils/renderDiffWithHighlighter.ts"],"sourcesContent":["import { type ChangeObject, diffChars, diffWordsWithSpace } from 'diff';\n\nimport { DEFAULT_COLLAPSED_CONTEXT_THRESHOLD } from '../constants';\nimport type {\n  DecorationItem,\n  DiffsHighlighter,\n  FileContents,\n  FileDiffMetadata,\n  ForceDiffPlainTextOptions,\n  LineDiffTypes,\n  LineInfo,\n  RenderDiffFilesResult,\n  RenderDiffOptions,\n  SupportedLanguages,\n  ThemedDiffResult,\n} from '../types';\nimport { appendItems } from './appendItems';\nimport { cleanLastNewline } from './cleanLastNewline';\nimport { getFiletypeFromFileName } from './getFiletypeFromFileName';\nimport { getHighlighterThemeStyles } from './getHighlighterThemeStyles';\nimport { getTokenOptions } from './getTokenOptions';\nimport { iterateOverDiff } from './iterateOverDiff';\nimport {\n  createDiffSpanDecoration,\n  pushOrJoinSpan,\n} from './parseDiffDecorations';\nimport { renderTokenLines } from './renderTokenLines';\nimport { setDeferredArrayItem } from './setDeferredArrayItem';\n\nconst DEFAULT_PLAIN_TEXT_OPTIONS: ForceDiffPlainTextOptions = {\n  forcePlainText: false,\n};\n\nexport function renderDiffWithHighlighter(\n  diff: FileDiffMetadata,\n  highlighter: DiffsHighlighter,\n  options: RenderDiffOptions,\n  {\n    forcePlainText,\n    lazyLineAST = false,\n    startingLine,\n    totalLines,\n    expandedHunks,\n    collapsedContextThreshold = DEFAULT_COLLAPSED_CONTEXT_THRESHOLD,\n  }: ForceDiffPlainTextOptions = DEFAULT_PLAIN_TEXT_OPTIONS\n): ThemedDiffResult {\n  if (forcePlainText) {\n    startingLine ??= 0;\n    totalLines ??= Infinity;\n  } else {\n    // Tokenization must include preceding lines to preserve lexical state.\n    startingLine = 0;\n    totalLines = Infinity;\n  }\n  const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;\n  lazyLineAST &&= !forcePlainText && !options.useTokenTransformer;\n  const baseThemeType =\n    typeof options.theme === 'string'\n      ? highlighter.getTheme(options.theme).type\n      : undefined;\n  const themeStyles = getHighlighterThemeStyles({\n    theme: options.theme,\n    highlighter,\n  });\n\n  // If we have a large file and we are rendering the WHOLE plain diff ast,\n  // then we should remove the lineDiffType to make sure things render quickly.\n  // For highlighted ASTs or windowed highlights, we should just inherit the\n  // setting\n  const lineDiffType =\n    forcePlainText &&\n    !isWindowedHighlight &&\n    (diff.unifiedLineCount > 1000 || diff.splitLineCount > 1000)\n      ? 'none'\n      : options.lineDiffType;\n\n  const code: RenderDiffFilesResult = {\n    deletionLines: [],\n    additionLines: [],\n  };\n\n  const { maxLineDiffLength } = options;\n  const shouldGroupAll = !forcePlainText && !diff.isPartial;\n  const expandedHunksForIteration = forcePlainText ? expandedHunks : undefined;\n  const buckets = new Map<number, RenderBucket>();\n  function getBucketForHunk(hunkIndex: number) {\n    const index = shouldGroupAll ? 0 : hunkIndex;\n    const bucket = buckets.get(index) ?? createBucket();\n    buckets.set(index, bucket);\n    return bucket;\n  }\n\n  function appendContent(\n    lineContent: string,\n    lineIndex: number,\n    segments: HighlightSegment[],\n    contentWrapper: FakeArrayType\n  ) {\n    if (isWindowedHighlight) {\n      let segment = segments.at(-1);\n      if (\n        segment == null ||\n        segment.targetIndex + segment.count !== lineIndex\n      ) {\n        segment = {\n          targetIndex: lineIndex,\n          originalOffset: contentWrapper.length,\n          count: 0,\n        };\n        segments.push(segment);\n      }\n      segment.count++;\n    }\n    contentWrapper.push(lineContent);\n  }\n\n  iterateOverDiff({\n    diff,\n    diffStyle: 'both',\n    startingLine,\n    totalLines,\n    expandedHunks: isWindowedHighlight ? expandedHunksForIteration : true,\n    collapsedContextThreshold,\n    callback: ({ hunkIndex, additionLine, deletionLine, type }) => {\n      const bucket = getBucketForHunk(hunkIndex);\n      const splitLineIndex =\n        additionLine != null\n          ? additionLine.splitLineIndex\n          : deletionLine.splitLineIndex;\n\n      if (type === 'change' && additionLine != null && deletionLine != null) {\n        computeLineDiffDecorations({\n          additionLine: diff.additionLines[additionLine.lineIndex],\n          deletionLine: diff.deletionLines[deletionLine.lineIndex],\n          deletionLineIndex: bucket.deletionContent.length,\n          additionLineIndex: bucket.additionContent.length,\n          deletionDecorations: bucket.deletionDecorations,\n          additionDecorations: bucket.additionDecorations,\n          lineDiffType,\n          maxLineDiffLength,\n        });\n      }\n\n      if (deletionLine != null) {\n        appendContent(\n          diff.deletionLines[deletionLine.lineIndex],\n          deletionLine.lineIndex,\n          bucket.deletionSegments,\n          bucket.deletionContent\n        );\n        bucket.deletionInfo.push({\n          type: type === 'change' ? 'change-deletion' : type,\n          lineNumber: deletionLine.lineNumber,\n          altLineNumber:\n            type === 'change'\n              ? undefined\n              : (additionLine.lineNumber ?? undefined),\n          lineIndex: `${deletionLine.unifiedLineIndex},${splitLineIndex}`,\n        });\n      }\n\n      if (additionLine != null) {\n        appendContent(\n          diff.additionLines[additionLine.lineIndex],\n          additionLine.lineIndex,\n          bucket.additionSegments,\n          bucket.additionContent\n        );\n        bucket.additionInfo.push({\n          type: type === 'change' ? 'change-addition' : type,\n          lineNumber: additionLine.lineNumber,\n          altLineNumber:\n            type === 'change'\n              ? undefined\n              : (deletionLine.lineNumber ?? undefined),\n          lineIndex: `${additionLine.unifiedLineIndex},${splitLineIndex}`,\n        });\n      }\n    },\n  });\n\n  for (const bucket of buckets.values()) {\n    if (\n      bucket.deletionContent.length === 0 &&\n      bucket.additionContent.length === 0\n    ) {\n      continue;\n    }\n\n    const deletionFile = {\n      name: diff.prevName ?? diff.name,\n      contents: bucket.deletionContent.value,\n    };\n    const additionFile = {\n      name: diff.name,\n      contents: bucket.additionContent.value,\n    };\n    const { deletionLines, additionLines } = renderTwoFiles({\n      deletionFile,\n      deletionInfo: bucket.deletionInfo,\n      deletionDecorations: bucket.deletionDecorations,\n\n      additionFile,\n      additionInfo: bucket.additionInfo,\n      additionDecorations: bucket.additionDecorations,\n\n      highlighter,\n      options,\n      languageOverride: forcePlainText ? 'text' : diff.lang,\n      lazyLineAST,\n    });\n\n    if (shouldGroupAll) {\n      code.deletionLines = deletionLines;\n      code.additionLines = additionLines;\n      continue;\n    }\n\n    for (const [target, lines, segments] of [\n      [code.deletionLines, deletionLines, bucket.deletionSegments],\n      [code.additionLines, additionLines, bucket.additionSegments],\n    ] as const) {\n      if (segments.length > 0) {\n        for (const segment of segments) {\n          for (let index = 0; index < segment.count; index++) {\n            target[segment.targetIndex + index] =\n              lines[segment.originalOffset + index];\n          }\n        }\n      } else if (lazyLineAST) {\n        const offset = target.length;\n        for (let index = 0; index < lines.length; index++) {\n          setDeferredArrayItem(target, offset + index, () => lines[index]);\n        }\n      } else {\n        appendItems(target, lines);\n      }\n    }\n  }\n\n  return { code, themeStyles, baseThemeType };\n}\n\n// The line indexes and decoration arrays a line diff writes its highlights to\ninterface LineDiffDecorationTargets {\n  deletionLineIndex: number;\n  additionLineIndex: number;\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n}\n\ninterface ProcessLineDiffProps extends LineDiffDecorationTargets {\n  deletionLine: string | undefined;\n  additionLine: string | undefined;\n  lineDiffType: LineDiffTypes;\n  maxLineDiffLength: number;\n}\n\n// Push one decoration per side covering the first through last changed\n// token, so each line only has one highlight. Offsets come straight from the\n// diff items, skipping the span arrays the other word modes build.\nfunction pushWordLineDecorations(\n  lineDiff: ChangeObject<string>[],\n  {\n    deletionLineIndex,\n    additionLineIndex,\n    deletionDecorations,\n    additionDecorations,\n  }: LineDiffDecorationTargets\n): void {\n  let deletionOffset = 0;\n  let additionOffset = 0;\n  let deletionStart = -1;\n  let deletionEnd = 0;\n  let additionStart = -1;\n  let additionEnd = 0;\n  for (const { value, added, removed } of lineDiff) {\n    if (removed) {\n      if (deletionStart === -1) deletionStart = deletionOffset;\n      deletionOffset += value.length;\n      deletionEnd = deletionOffset;\n    } else if (added) {\n      if (additionStart === -1) additionStart = additionOffset;\n      additionOffset += value.length;\n      additionEnd = additionOffset;\n    } else {\n      deletionOffset += value.length;\n      additionOffset += value.length;\n    }\n  }\n  if (deletionStart !== -1) {\n    deletionDecorations.push(\n      createDiffSpanDecoration({\n        line: deletionLineIndex,\n        spanStart: deletionStart,\n        spanLength: deletionEnd - deletionStart,\n      })\n    );\n  }\n  if (additionStart !== -1) {\n    additionDecorations.push(\n      createDiffSpanDecoration({\n        line: additionLineIndex,\n        spanStart: additionStart,\n        spanLength: additionEnd - additionStart,\n      })\n    );\n  }\n}\n\nfunction computeLineDiffDecorations({\n  deletionLine,\n  additionLine,\n  deletionLineIndex,\n  additionLineIndex,\n  deletionDecorations,\n  additionDecorations,\n  lineDiffType,\n  maxLineDiffLength,\n}: ProcessLineDiffProps) {\n  if (deletionLine == null || additionLine == null || lineDiffType === 'none') {\n    return;\n  }\n  deletionLine = cleanLastNewline(deletionLine);\n  additionLine = cleanLastNewline(additionLine);\n  // If we have really long lines, we probably shouldn't compute diffs on them.\n  if (\n    deletionLine.length > maxLineDiffLength ||\n    additionLine.length > maxLineDiffLength\n  ) {\n    return;\n  }\n  // NOTE(amadeus): Because we visually trim trailing newlines when rendering,\n  // we also gotta make sure the diff parsing doesn't include the newline\n  // character that could be there...\n  const lineDiff =\n    lineDiffType === 'char'\n      ? diffChars(deletionLine, additionLine)\n      : diffWordsWithSpace(deletionLine, additionLine);\n  if (lineDiffType === 'word-line') {\n    pushWordLineDecorations(lineDiff, {\n      deletionLineIndex,\n      additionLineIndex,\n      deletionDecorations,\n      additionDecorations,\n    });\n    return;\n  }\n  const deletionSpans: [0 | 1, string][] = [];\n  const additionSpans: [0 | 1, string][] = [];\n  const enableJoin = lineDiffType === 'word-alt';\n  const lastItem = lineDiff.at(-1);\n  for (const item of lineDiff) {\n    const isLastItem = item === lastItem;\n    if (!item.added && !item.removed) {\n      pushOrJoinSpan({\n        item,\n        arr: deletionSpans,\n        enableJoin,\n        isNeutral: true,\n        isLastItem,\n      });\n      pushOrJoinSpan({\n        item,\n        arr: additionSpans,\n        enableJoin,\n        isNeutral: true,\n        isLastItem,\n      });\n    } else if (item.removed) {\n      pushOrJoinSpan({ item, arr: deletionSpans, enableJoin, isLastItem });\n    } else {\n      pushOrJoinSpan({ item, arr: additionSpans, enableJoin, isLastItem });\n    }\n  }\n  let spanIndex = 0;\n  for (const span of deletionSpans) {\n    if (span[0] === 1) {\n      deletionDecorations.push(\n        createDiffSpanDecoration({\n          line: deletionLineIndex,\n          spanStart: spanIndex,\n          spanLength: span[1].length,\n        })\n      );\n    }\n    spanIndex += span[1].length;\n  }\n  spanIndex = 0;\n  for (const span of additionSpans) {\n    if (span[0] === 1) {\n      additionDecorations.push(\n        createDiffSpanDecoration({\n          line: additionLineIndex,\n          spanStart: spanIndex,\n          spanLength: span[1].length,\n        })\n      );\n    }\n    spanIndex += span[1].length;\n  }\n}\n\ninterface HighlightSegment {\n  // The where the highlighted region starts\n  originalOffset: number;\n  // Where to place the highlighted line in RenderDiffFilesResult\n  targetIndex: number;\n  // Number of highlighted lines\n  count: number;\n}\n\ninterface FakeArrayType {\n  push(value: string): void;\n  value: string;\n  length: number;\n}\n\ninterface RenderBucket {\n  deletionContent: FakeArrayType;\n  additionContent: FakeArrayType;\n  deletionInfo: (LineInfo | undefined)[];\n  additionInfo: (LineInfo | undefined)[];\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n  deletionSegments: HighlightSegment[];\n  additionSegments: HighlightSegment[];\n}\n\nfunction createBucket(): RenderBucket {\n  return {\n    deletionContent: {\n      push(value: string) {\n        this.value += value;\n        this.length++;\n      },\n      value: '',\n      length: 0,\n    },\n    additionContent: {\n      push(value: string) {\n        this.value += value;\n        this.length++;\n      },\n      value: '',\n      length: 0,\n    },\n    deletionInfo: [],\n    additionInfo: [],\n    deletionDecorations: [],\n    additionDecorations: [],\n    deletionSegments: [],\n    additionSegments: [],\n  };\n}\n\ninterface RenderTwoFilesProps {\n  lazyLineAST: boolean;\n  deletionFile: FileContents;\n  additionFile: FileContents;\n  deletionInfo: (LineInfo | undefined)[];\n  additionInfo: (LineInfo | undefined)[];\n  deletionDecorations: DecorationItem[];\n  additionDecorations: DecorationItem[];\n  options: RenderDiffOptions;\n  highlighter: DiffsHighlighter;\n  languageOverride: SupportedLanguages | undefined;\n}\n\nfunction renderTwoFiles({\n  lazyLineAST,\n  deletionFile,\n  additionFile,\n  deletionInfo,\n  additionInfo,\n  highlighter,\n  deletionDecorations,\n  additionDecorations,\n  languageOverride,\n  options: { theme, useTokenTransformer, tokenizeMaxLineLength },\n}: RenderTwoFilesProps): RenderDiffFilesResult {\n  const renderFile = (\n    file: FileContents,\n    lineInfo: (LineInfo | undefined)[],\n    decorations: DecorationItem[]\n  ): RenderDiffFilesResult['additionLines'] => {\n    if (file.contents === '') return [];\n    return renderTokenLines(\n      highlighter.codeToTokens(\n        cleanLastNewline(file.contents),\n        getTokenOptions(\n          languageOverride ?? getFiletypeFromFileName(file.name),\n          theme,\n          tokenizeMaxLineLength\n        )\n      ).tokens,\n      {\n        state: { lineInfo },\n        useTokenTransformer,\n        decorations,\n        lazyLineAST,\n        cacheHtmlStyles: highlighter.name === 'highlights',\n      }\n    );\n  };\n\n  return {\n    deletionLines: renderFile(deletionFile, deletionInfo, deletionDecorations),\n    additionLines: renderFile(additionFile, additionInfo, additionDecorations),\n  };\n}\n"],"mappings":";;;;;;;;;;;;AA6BA,MAAM,6BAAwD,EAC5D,gBAAgB,MAClB;AAEA,SAAgB,0BACd,MACA,aACA,SACA,EACE,gBACA,cAAc,OACd,cACA,YACA,eACA,4BAAA,MAC6B,4BACb;CAClB,IAAI,gBAAgB;EAClB,iBAAiB;EACjB,eAAe;CACjB,OAAO;EAEL,eAAe;EACf,aAAa;CACf;CACA,MAAM,sBAAsB,eAAe,KAAK,aAAa;CAC7D,gBAAgB,CAAC,kBAAkB,CAAC,QAAQ;CAC5C,MAAM,gBACJ,OAAO,QAAQ,UAAU,WACrB,YAAY,SAAS,QAAQ,KAAK,CAAC,CAAC,OACpC,KAAA;CACN,MAAM,cAAc,0BAA0B;EAC5C,OAAO,QAAQ;EACf;CACF,CAAC;CAMD,MAAM,eACJ,kBACA,CAAC,wBACA,KAAK,mBAAmB,OAAQ,KAAK,iBAAiB,OACnD,SACA,QAAQ;CAEd,MAAM,OAA8B;EAClC,eAAe,CAAC;EAChB,eAAe,CAAC;CAClB;CAEA,MAAM,EAAE,sBAAsB;CAC9B,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,KAAK;CAChD,MAAM,4BAA4B,iBAAiB,gBAAgB,KAAA;CACnE,MAAM,0BAAU,IAAI,IAA0B;CAC9C,SAAS,iBAAiB,WAAmB;EAC3C,MAAM,QAAQ,iBAAiB,IAAI;EACnC,MAAM,SAAS,QAAQ,IAAI,KAAK,KAAK,aAAa;EAClD,QAAQ,IAAI,OAAO,MAAM;EACzB,OAAO;CACT;CAEA,SAAS,cACP,aACA,WACA,UACA,gBACA;EACA,IAAI,qBAAqB;GACvB,IAAI,UAAU,SAAS,GAAG,EAAE;GAC5B,IACE,WAAW,QACX,QAAQ,cAAc,QAAQ,UAAU,WACxC;IACA,UAAU;KACR,aAAa;KACb,gBAAgB,eAAe;KAC/B,OAAO;IACT;IACA,SAAS,KAAK,OAAO;GACvB;GACA,QAAQ;EACV;EACA,eAAe,KAAK,WAAW;CACjC;CAEA,gBAAgB;EACd;EACA,WAAW;EACX;EACA;EACA,eAAe,sBAAsB,4BAA4B;EACjE;EACA,WAAW,EAAE,WAAW,cAAc,cAAc,WAAW;GAC7D,MAAM,SAAS,iBAAiB,SAAS;GACzC,MAAM,iBACJ,gBAAgB,OACZ,aAAa,iBACb,aAAa;GAEnB,IAAI,SAAS,YAAY,gBAAgB,QAAQ,gBAAgB,MAC/D,2BAA2B;IACzB,cAAc,KAAK,cAAc,aAAa;IAC9C,cAAc,KAAK,cAAc,aAAa;IAC9C,mBAAmB,OAAO,gBAAgB;IAC1C,mBAAmB,OAAO,gBAAgB;IAC1C,qBAAqB,OAAO;IAC5B,qBAAqB,OAAO;IAC5B;IACA;GACF,CAAC;GAGH,IAAI,gBAAgB,MAAM;IACxB,cACE,KAAK,cAAc,aAAa,YAChC,aAAa,WACb,OAAO,kBACP,OAAO,eACT;IACA,OAAO,aAAa,KAAK;KACvB,MAAM,SAAS,WAAW,oBAAoB;KAC9C,YAAY,aAAa;KACzB,eACE,SAAS,WACL,KAAA,IACC,aAAa,cAAc,KAAA;KAClC,WAAW,GAAG,aAAa,iBAAiB,GAAG;IACjD,CAAC;GACH;GAEA,IAAI,gBAAgB,MAAM;IACxB,cACE,KAAK,cAAc,aAAa,YAChC,aAAa,WACb,OAAO,kBACP,OAAO,eACT;IACA,OAAO,aAAa,KAAK;KACvB,MAAM,SAAS,WAAW,oBAAoB;KAC9C,YAAY,aAAa;KACzB,eACE,SAAS,WACL,KAAA,IACC,aAAa,cAAc,KAAA;KAClC,WAAW,GAAG,aAAa,iBAAiB,GAAG;IACjD,CAAC;GACH;EACF;CACF,CAAC;CAED,KAAK,MAAM,UAAU,QAAQ,OAAO,GAAG;EACrC,IACE,OAAO,gBAAgB,WAAW,KAClC,OAAO,gBAAgB,WAAW,GAElC;EAGF,MAAM,eAAe;GACnB,MAAM,KAAK,YAAY,KAAK;GAC5B,UAAU,OAAO,gBAAgB;EACnC;EACA,MAAM,eAAe;GACnB,MAAM,KAAK;GACX,UAAU,OAAO,gBAAgB;EACnC;EACA,MAAM,EAAE,eAAe,kBAAkB,eAAe;GACtD;GACA,cAAc,OAAO;GACrB,qBAAqB,OAAO;GAE5B;GACA,cAAc,OAAO;GACrB,qBAAqB,OAAO;GAE5B;GACA;GACA,kBAAkB,iBAAiB,SAAS,KAAK;GACjD;EACF,CAAC;EAED,IAAI,gBAAgB;GAClB,KAAK,gBAAgB;GACrB,KAAK,gBAAgB;GACrB;EACF;EAEA,KAAK,MAAM,CAAC,QAAQ,OAAO,aAAa,CACtC;GAAC,KAAK;GAAe;GAAe,OAAO;EAAgB,GAC3D;GAAC,KAAK;GAAe;GAAe,OAAO;EAAgB,CAC7D,GACE,IAAI,SAAS,SAAS,GACpB,KAAK,MAAM,WAAW,UACpB,KAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,OAAO,SACzC,OAAO,QAAQ,cAAc,SAC3B,MAAM,QAAQ,iBAAiB;OAGhC,IAAI,aAAa;GACtB,MAAM,SAAS,OAAO;GACtB,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SACxC,qBAAqB,QAAQ,SAAS,aAAa,MAAM,MAAM;EAEnE,OACE,YAAY,QAAQ,KAAK;CAG/B;CAEA,OAAO;EAAE;EAAM;EAAa;CAAc;AAC5C;AAoBA,SAAS,wBACP,UACA,EACE,mBACA,mBACA,qBACA,uBAEI;CACN,IAAI,iBAAiB;CACrB,IAAI,iBAAiB;CACrB,IAAI,gBAAgB;CACpB,IAAI,cAAc;CAClB,IAAI,gBAAgB;CACpB,IAAI,cAAc;CAClB,KAAK,MAAM,EAAE,OAAO,OAAO,aAAa,UACtC,IAAI,SAAS;EACX,IAAI,kBAAkB,IAAI,gBAAgB;EAC1C,kBAAkB,MAAM;EACxB,cAAc;CAChB,OAAO,IAAI,OAAO;EAChB,IAAI,kBAAkB,IAAI,gBAAgB;EAC1C,kBAAkB,MAAM;EACxB,cAAc;CAChB,OAAO;EACL,kBAAkB,MAAM;EACxB,kBAAkB,MAAM;CAC1B;CAEF,IAAI,kBAAkB,IACpB,oBAAoB,KAClB,yBAAyB;EACvB,MAAM;EACN,WAAW;EACX,YAAY,cAAc;CAC5B,CAAC,CACH;CAEF,IAAI,kBAAkB,IACpB,oBAAoB,KAClB,yBAAyB;EACvB,MAAM;EACN,WAAW;EACX,YAAY,cAAc;CAC5B,CAAC,CACH;AAEJ;AAEA,SAAS,2BAA2B,EAClC,cACA,cACA,mBACA,mBACA,qBACA,qBACA,cACA,qBACuB;CACvB,IAAI,gBAAgB,QAAQ,gBAAgB,QAAQ,iBAAiB,QACnE;CAEF,eAAe,iBAAiB,YAAY;CAC5C,eAAe,iBAAiB,YAAY;CAE5C,IACE,aAAa,SAAS,qBACtB,aAAa,SAAS,mBAEtB;CAKF,MAAM,WACJ,iBAAiB,SACb,UAAU,cAAc,YAAY,IACpC,mBAAmB,cAAc,YAAY;CACnD,IAAI,iBAAiB,aAAa;EAChC,wBAAwB,UAAU;GAChC;GACA;GACA;GACA;EACF,CAAC;EACD;CACF;CACA,MAAM,gBAAmC,CAAC;CAC1C,MAAM,gBAAmC,CAAC;CAC1C,MAAM,aAAa,iBAAiB;CACpC,MAAM,WAAW,SAAS,GAAG,EAAE;CAC/B,KAAK,MAAM,QAAQ,UAAU;EAC3B,MAAM,aAAa,SAAS;EAC5B,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,SAAS;GAChC,eAAe;IACb;IACA,KAAK;IACL;IACA,WAAW;IACX;GACF,CAAC;GACD,eAAe;IACb;IACA,KAAK;IACL;IACA,WAAW;IACX;GACF,CAAC;EACH,OAAO,IAAI,KAAK,SACd,eAAe;GAAE;GAAM,KAAK;GAAe;GAAY;EAAW,CAAC;OAEnE,eAAe;GAAE;GAAM,KAAK;GAAe;GAAY;EAAW,CAAC;CAEvE;CACA,IAAI,YAAY;CAChB,KAAK,MAAM,QAAQ,eAAe;EAChC,IAAI,KAAK,OAAO,GACd,oBAAoB,KAClB,yBAAyB;GACvB,MAAM;GACN,WAAW;GACX,YAAY,KAAK,EAAE,CAAC;EACtB,CAAC,CACH;EAEF,aAAa,KAAK,EAAE,CAAC;CACvB;CACA,YAAY;CACZ,KAAK,MAAM,QAAQ,eAAe;EAChC,IAAI,KAAK,OAAO,GACd,oBAAoB,KAClB,yBAAyB;GACvB,MAAM;GACN,WAAW;GACX,YAAY,KAAK,EAAE,CAAC;EACtB,CAAC,CACH;EAEF,aAAa,KAAK,EAAE,CAAC;CACvB;AACF;AA4BA,SAAS,eAA6B;CACpC,OAAO;EACL,iBAAiB;GACf,KAAK,OAAe;IAClB,KAAK,SAAS;IACd,KAAK;GACP;GACA,OAAO;GACP,QAAQ;EACV;EACA,iBAAiB;GACf,KAAK,OAAe;IAClB,KAAK,SAAS;IACd,KAAK;GACP;GACA,OAAO;GACP,QAAQ;EACV;EACA,cAAc,CAAC;EACf,cAAc,CAAC;EACf,qBAAqB,CAAC;EACtB,qBAAqB,CAAC;EACtB,kBAAkB,CAAC;EACnB,kBAAkB,CAAC;CACrB;AACF;AAeA,SAAS,eAAe,EACtB,aACA,cACA,cACA,cACA,cACA,aACA,qBACA,qBACA,kBACA,SAAS,EAAE,OAAO,qBAAqB,2BACM;CAC7C,MAAM,cACJ,MACA,UACA,gBAC2C;EAC3C,IAAI,KAAK,aAAa,IAAI,OAAO,CAAC;EAClC,OAAO,iBACL,YAAY,aACV,iBAAiB,KAAK,QAAQ,GAC9B,gBACE,oBAAoB,wBAAwB,KAAK,IAAI,GACrD,OACA,qBACF,CACF,CAAC,CAAC,QACF;GACE,OAAO,EAAE,SAAS;GAClB;GACA;GACA;GACA,iBAAiB,YAAY,SAAS;EACxC,CACF;CACF;CAEA,OAAO;EACL,eAAe,WAAW,cAAc,cAAc,mBAAmB;EACzE,eAAe,WAAW,cAAc,cAAc,mBAAmB;CAC3E;AACF"}

package/dist/utils/renderFileWithHighlighter.d.ts

@@ -1,6 +1,7 @@
-import { DiffsHighlighter, FileContents, ForceFilePlainTextOptions, RenderFileOptions, ThemedFileResult } from "../types.js";
+import { DiffsHighlighter } from "../highlighter/DiffsHighlighter.js";
+import { FileContents, ForceFilePlainTextOptions, RenderFileOptions, ThemedFileResult } from "../types.js";
 //#region src/utils/renderFileWithHighlighter.d.ts
-declare function renderFileWithHighlighter(file: FileContents, highlighter: DiffsHighlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }: RenderFileOptions, { forcePlainText, startingLine, totalLines, lines }?: ForceFilePlainTextOptions): ThemedFileResult;
+declare function renderFileWithHighlighter(file: FileContents, highlighter: DiffsHighlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }: RenderFileOptions, { forcePlainText, lazyLineAST, startingLine, totalLines, lines }?: ForceFilePlainTextOptions): ThemedFileResult;
 //#endregion
 export { renderFileWithHighlighter };
 //# sourceMappingURL=renderFileWithHighlighter.d.ts.map

package/dist/utils/renderFileWithHighlighter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"renderFileWithHighlighter.d.ts","names":[],"sources":["../../src/utils/renderFileWithHighlighter.ts"],"mappings":";;iBAqBgB,0BACd,MAAM,cACN,aAAa,oBACX,OAAO,uBAAuB,uBAAuB,qBAErD,gBACA,cACA,YACA,UACC,4BACF"}
+{"version":3,"file":"renderFileWithHighlighter.d.ts","names":[],"sources":["../../src/utils/renderFileWithHighlighter.ts"],"mappings":";;;iBAmBgB,0BACd,MAAM,cACN,aAAa,oBACX,OAAO,uBAAuB,uBAAuB,qBAErD,gBACA,aACA,cACA,YACA,UACC,4BACF"}

package/dist/utils/renderFileWithHighlighter.js

@@ -1,13 +1,12 @@
 import { linesFromFileContents } from "./computeFileOffsets.js";
 import { getFiletypeFromFileName } from "./getFiletypeFromFileName.js";
 import { appendItems } from "./appendItems.js";
-import { createTransformerWithState } from "./createTransformerWithState.js";
-import { formatCSSVariablePrefix } from "./formatCSSVariablePrefix.js";
 import { getHighlighterThemeStyles } from "./getHighlighterThemeStyles.js";
-import { getLineNodes } from "./getLineNodes.js";
+import { getTokenOptions } from "./getTokenOptions.js";
+import { renderTokenLines } from "./renderTokenLines.js";
 //#region src/utils/renderFileWithHighlighter.ts
 const DEFAULT_PLAIN_TEXT_OPTIONS = { forcePlainText: false };
-function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
+function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLength, useTokenTransformer }, { forcePlainText, lazyLineAST = false, startingLine, totalLines, lines } = DEFAULT_PLAIN_TEXT_OPTIONS) {
 	if (forcePlainText) {
 		startingLine ??= 0;
 		totalLines ??= Infinity;
@@ -16,7 +15,7 @@ function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLe
 		totalLines = Infinity;
 	}
 	const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;
-	const { state, transformers } = createTransformerWithState(useTokenTransformer);
+	const state = { lineInfo: [] };
 	const lang = forcePlainText ? "text" : file.lang ?? getFiletypeFromFileName(file.name);
 	const baseThemeType = typeof theme === "string" ? highlighter.getTheme(theme).type : void 0;
 	const themeStyles = getHighlighterThemeStyles({
@@ -28,27 +27,12 @@ function renderFileWithHighlighter(file, highlighter, { theme, tokenizeMaxLineLe
 		lineIndex: shikiLineNumber - 1 + startingLine,
 		lineNumber: shikiLineNumber + startingLine
 	});
-	const hastConfig = (() => {
-		if (typeof theme === "string") return {
-			lang,
-			theme,
-			transformers,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeMaxLineLength,
-			tokenizeTimeLimit: 0
-		};
-		return {
-			lang,
-			themes: theme,
-			transformers,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeMaxLineLength,
-			tokenizeTimeLimit: 0
-		};
-	})();
-	const highlightedLines = getLineNodes(highlighter.codeToHast(normalizeHighlightLineEndings(isWindowedHighlight ? extractWindowedFileContent(lines ?? linesFromFileContents(file.contents), startingLine, totalLines) : file.contents), hastConfig));
+	const highlightedLines = renderTokenLines(highlighter.codeToTokens(normalizeHighlightLineEndings(isWindowedHighlight ? extractWindowedFileContent(lines ?? linesFromFileContents(file.contents), startingLine, totalLines) : file.contents), getTokenOptions(lang, theme, tokenizeMaxLineLength)).tokens, {
+		state,
+		useTokenTransformer,
+		lazyLineAST: lazyLineAST && !forcePlainText && !useTokenTransformer,
+		cacheHtmlStyles: highlighter.name === "highlights"
+	});
 	const code = isWindowedHighlight ? new Array(startingLine) : highlightedLines;
 	if (isWindowedHighlight) appendItems(code, highlightedLines);
 	return {

package/dist/utils/renderFileWithHighlighter.js.map

@@ -1 +1 @@
-{"version":3,"file":"renderFileWithHighlighter.js","names":[],"sources":["../../src/utils/renderFileWithHighlighter.ts"],"sourcesContent":["import type {\n  CodeToHastOptions,\n  DiffsHighlighter,\n  DiffsThemeNames,\n  FileContents,\n  ForceFilePlainTextOptions,\n  RenderFileOptions,\n  ThemedFileResult,\n} from '../types';\nimport { appendItems } from './appendItems';\nimport { linesFromFileContents } from './computeFileOffsets';\nimport { createTransformerWithState } from './createTransformerWithState';\nimport { formatCSSVariablePrefix } from './formatCSSVariablePrefix';\nimport { getFiletypeFromFileName } from './getFiletypeFromFileName';\nimport { getHighlighterThemeStyles } from './getHighlighterThemeStyles';\nimport { getLineNodes } from './getLineNodes';\n\nconst DEFAULT_PLAIN_TEXT_OPTIONS: ForceFilePlainTextOptions = {\n  forcePlainText: false,\n};\n\nexport function renderFileWithHighlighter(\n  file: FileContents,\n  highlighter: DiffsHighlighter,\n  { theme, tokenizeMaxLineLength, useTokenTransformer }: RenderFileOptions,\n  {\n    forcePlainText,\n    startingLine,\n    totalLines,\n    lines,\n  }: ForceFilePlainTextOptions = DEFAULT_PLAIN_TEXT_OPTIONS\n): ThemedFileResult {\n  if (forcePlainText) {\n    startingLine ??= 0;\n    totalLines ??= Infinity;\n  } else {\n    // If we aren't forcing plain text, then we intentionally do not support\n    // ranges for highlighting as that could break the syntax highlighting, we\n    // we override any values that may have been passed in.  Maybe one day we\n    // warn about this?\n    startingLine = 0;\n    totalLines = Infinity;\n  }\n  const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;\n  const { state, transformers } =\n    createTransformerWithState(useTokenTransformer);\n  const lang = forcePlainText\n    ? 'text'\n    : (file.lang ?? getFiletypeFromFileName(file.name));\n  const baseThemeType =\n    typeof theme === 'string' ? highlighter.getTheme(theme).type : undefined;\n  const themeStyles = getHighlighterThemeStyles({\n    theme,\n    highlighter,\n  });\n  state.lineInfo = (shikiLineNumber: number) => ({\n    type: 'context',\n    lineIndex: shikiLineNumber - 1 + startingLine,\n    lineNumber: shikiLineNumber + startingLine,\n  });\n  // tokenizeTimeLimit: 0 disables shiki's silent 500ms-per-line tokenization\n  // abort. When it trips (slow devices, cold JS-regex-engine compile), the\n  // rest of the line collapses to the enclosing scope's color — and since\n  // dual-theme rendering tokenizes per theme, the first (dark) pass can smear\n  // while the warm second (light) pass stays correct. Pathological content is\n  // already guarded by tokenizeMaxLineLength, which renders long lines plain.\n  const hastConfig: CodeToHastOptions<DiffsThemeNames> = (() => {\n    if (typeof theme === 'string') {\n      return {\n        lang,\n        theme,\n        transformers,\n        defaultColor: false,\n        cssVariablePrefix: formatCSSVariablePrefix('token'),\n        tokenizeMaxLineLength,\n        tokenizeTimeLimit: 0,\n      };\n    }\n    return {\n      lang,\n      themes: theme,\n      transformers,\n      defaultColor: false,\n      cssVariablePrefix: formatCSSVariablePrefix('token'),\n      tokenizeMaxLineLength,\n      tokenizeTimeLimit: 0,\n    };\n  })();\n  const highlightedLines = getLineNodes(\n    highlighter.codeToHast(\n      normalizeHighlightLineEndings(\n        isWindowedHighlight\n          ? extractWindowedFileContent(\n              lines ?? linesFromFileContents(file.contents),\n              startingLine,\n              totalLines\n            )\n          : file.contents\n      ),\n      hastConfig\n    )\n  );\n\n  // Create sparse array for windowed rendering\n  const code = isWindowedHighlight ? new Array(startingLine) : highlightedLines;\n  if (isWindowedHighlight) {\n    appendItems(code, highlightedLines);\n  }\n\n  return { code, themeStyles, baseThemeType };\n}\n\n// Shiki does not treat a lone carriage return as a line break. Normalize only\n// the text sent to the highlighter so its output stays aligned with the file\n// model while the original document retains its line endings.\nfunction normalizeHighlightLineEndings(contents: string): string {\n  return contents.replace(/\\r(?!\\n)/g, '\\n');\n}\n\nfunction extractWindowedFileContent(\n  lines: string[],\n  startingLine: number,\n  totalLines: number\n): string {\n  if (lines.length === 0) {\n    return '';\n  }\n  const endLine = Math.min(startingLine + totalLines, lines.length);\n  return lines.slice(startingLine, endLine).join('');\n}\n"],"mappings":";;;;;;;;AAiBA,MAAM,6BAAwD,EAC5D,gBAAgB,MAClB;AAEA,SAAgB,0BACd,MACA,aACA,EAAE,OAAO,uBAAuB,uBAChC,EACE,gBACA,cACA,YACA,UAC6B,4BACb;CAClB,IAAI,gBAAgB;EAClB,iBAAiB;EACjB,eAAe;CACjB,OAAO;EAKL,eAAe;EACf,aAAa;CACf;CACA,MAAM,sBAAsB,eAAe,KAAK,aAAa;CAC7D,MAAM,EAAE,OAAO,iBACb,2BAA2B,mBAAmB;CAChD,MAAM,OAAO,iBACT,SACC,KAAK,QAAQ,wBAAwB,KAAK,IAAI;CACnD,MAAM,gBACJ,OAAO,UAAU,WAAW,YAAY,SAAS,KAAK,CAAC,CAAC,OAAO,KAAA;CACjE,MAAM,cAAc,0BAA0B;EAC5C;EACA;CACF,CAAC;CACD,MAAM,YAAY,qBAA6B;EAC7C,MAAM;EACN,WAAW,kBAAkB,IAAI;EACjC,YAAY,kBAAkB;CAChC;CAOA,MAAM,oBAAwD;EAC5D,IAAI,OAAO,UAAU,UACnB,OAAO;GACL;GACA;GACA;GACA,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD;GACA,mBAAmB;EACrB;EAEF,OAAO;GACL;GACA,QAAQ;GACR;GACA,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD;GACA,mBAAmB;EACrB;CACF,EAAA,CAAG;CACH,MAAM,mBAAmB,aACvB,YAAY,WACV,8BACE,sBACI,2BACE,SAAS,sBAAsB,KAAK,QAAQ,GAC5C,cACA,UACF,IACA,KAAK,QACX,GACA,UACF,CACF;CAGA,MAAM,OAAO,sBAAsB,IAAI,MAAM,YAAY,IAAI;CAC7D,IAAI,qBACF,YAAY,MAAM,gBAAgB;CAGpC,OAAO;EAAE;EAAM;EAAa;CAAc;AAC5C;AAKA,SAAS,8BAA8B,UAA0B;CAC/D,OAAO,SAAS,QAAQ,aAAa,IAAI;AAC3C;AAEA,SAAS,2BACP,OACA,cACA,YACQ;CACR,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,UAAU,KAAK,IAAI,eAAe,YAAY,MAAM,MAAM;CAChE,OAAO,MAAM,MAAM,cAAc,OAAO,CAAC,CAAC,KAAK,EAAE;AACnD"}
+{"version":3,"file":"renderFileWithHighlighter.js","names":[],"sources":["../../src/utils/renderFileWithHighlighter.ts"],"sourcesContent":["import type {\n  DiffsHighlighter,\n  FileContents,\n  ForceFilePlainTextOptions,\n  RenderFileOptions,\n  SharedRenderState,\n  ThemedFileResult,\n} from '../types';\nimport { appendItems } from './appendItems';\nimport { linesFromFileContents } from './computeFileOffsets';\nimport { getFiletypeFromFileName } from './getFiletypeFromFileName';\nimport { getHighlighterThemeStyles } from './getHighlighterThemeStyles';\nimport { getTokenOptions } from './getTokenOptions';\nimport { renderTokenLines } from './renderTokenLines';\n\nconst DEFAULT_PLAIN_TEXT_OPTIONS: ForceFilePlainTextOptions = {\n  forcePlainText: false,\n};\n\nexport function renderFileWithHighlighter(\n  file: FileContents,\n  highlighter: DiffsHighlighter,\n  { theme, tokenizeMaxLineLength, useTokenTransformer }: RenderFileOptions,\n  {\n    forcePlainText,\n    lazyLineAST = false,\n    startingLine,\n    totalLines,\n    lines,\n  }: ForceFilePlainTextOptions = DEFAULT_PLAIN_TEXT_OPTIONS\n): ThemedFileResult {\n  if (forcePlainText) {\n    startingLine ??= 0;\n    totalLines ??= Infinity;\n  } else {\n    // Tokenization must include preceding lines to preserve lexical state.\n    startingLine = 0;\n    totalLines = Infinity;\n  }\n  const isWindowedHighlight = startingLine > 0 || totalLines < Infinity;\n  const state: SharedRenderState = { lineInfo: [] };\n  const lang = forcePlainText\n    ? 'text'\n    : (file.lang ?? getFiletypeFromFileName(file.name));\n  const baseThemeType =\n    typeof theme === 'string' ? highlighter.getTheme(theme).type : undefined;\n  const themeStyles = getHighlighterThemeStyles({\n    theme,\n    highlighter,\n  });\n  state.lineInfo = (shikiLineNumber: number) => ({\n    type: 'context',\n    lineIndex: shikiLineNumber - 1 + startingLine,\n    lineNumber: shikiLineNumber + startingLine,\n  });\n  const highlightedLines = renderTokenLines(\n    highlighter.codeToTokens(\n      normalizeHighlightLineEndings(\n        isWindowedHighlight\n          ? extractWindowedFileContent(\n              lines ?? linesFromFileContents(file.contents),\n              startingLine,\n              totalLines\n            )\n          : file.contents\n      ),\n      getTokenOptions(lang, theme, tokenizeMaxLineLength)\n    ).tokens,\n    {\n      state,\n      useTokenTransformer,\n      lazyLineAST: lazyLineAST && !forcePlainText && !useTokenTransformer,\n      cacheHtmlStyles: highlighter.name === 'highlights',\n    }\n  );\n\n  // Create sparse array for windowed rendering\n  const code = isWindowedHighlight ? new Array(startingLine) : highlightedLines;\n  if (isWindowedHighlight) {\n    appendItems(code, highlightedLines);\n  }\n\n  return { code, themeStyles, baseThemeType };\n}\n\n// Shiki does not treat a lone carriage return as a line break. Normalize only\n// the text sent to the highlighter so its output stays aligned with the file\n// model while the original document retains its line endings.\nfunction normalizeHighlightLineEndings(contents: string): string {\n  return contents.replace(/\\r(?!\\n)/g, '\\n');\n}\n\nfunction extractWindowedFileContent(\n  lines: string[],\n  startingLine: number,\n  totalLines: number\n): string {\n  if (lines.length === 0) {\n    return '';\n  }\n  const endLine = Math.min(startingLine + totalLines, lines.length);\n  return lines.slice(startingLine, endLine).join('');\n}\n"],"mappings":";;;;;;;AAeA,MAAM,6BAAwD,EAC5D,gBAAgB,MAClB;AAEA,SAAgB,0BACd,MACA,aACA,EAAE,OAAO,uBAAuB,uBAChC,EACE,gBACA,cAAc,OACd,cACA,YACA,UAC6B,4BACb;CAClB,IAAI,gBAAgB;EAClB,iBAAiB;EACjB,eAAe;CACjB,OAAO;EAEL,eAAe;EACf,aAAa;CACf;CACA,MAAM,sBAAsB,eAAe,KAAK,aAAa;CAC7D,MAAM,QAA2B,EAAE,UAAU,CAAC,EAAE;CAChD,MAAM,OAAO,iBACT,SACC,KAAK,QAAQ,wBAAwB,KAAK,IAAI;CACnD,MAAM,gBACJ,OAAO,UAAU,WAAW,YAAY,SAAS,KAAK,CAAC,CAAC,OAAO,KAAA;CACjE,MAAM,cAAc,0BAA0B;EAC5C;EACA;CACF,CAAC;CACD,MAAM,YAAY,qBAA6B;EAC7C,MAAM;EACN,WAAW,kBAAkB,IAAI;EACjC,YAAY,kBAAkB;CAChC;CACA,MAAM,mBAAmB,iBACvB,YAAY,aACV,8BACE,sBACI,2BACE,SAAS,sBAAsB,KAAK,QAAQ,GAC5C,cACA,UACF,IACA,KAAK,QACX,GACA,gBAAgB,MAAM,OAAO,qBAAqB,CACpD,CAAC,CAAC,QACF;EACE;EACA;EACA,aAAa,eAAe,CAAC,kBAAkB,CAAC;EAChD,iBAAiB,YAAY,SAAS;CACxC,CACF;CAGA,MAAM,OAAO,sBAAsB,IAAI,MAAM,YAAY,IAAI;CAC7D,IAAI,qBACF,YAAY,MAAM,gBAAgB;CAGpC,OAAO;EAAE;EAAM;EAAa;CAAc;AAC5C;AAKA,SAAS,8BAA8B,UAA0B;CAC/D,OAAO,SAAS,QAAQ,aAAa,IAAI;AAC3C;AAEA,SAAS,2BACP,OACA,cACA,YACQ;CACR,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,UAAU,KAAK,IAAI,eAAe,YAAY,MAAM,MAAM;CAChE,OAAO,MAAM,MAAM,cAAc,OAAO,CAAC,CAAC,KAAK,EAAE;AACnD"}

package/dist/utils/renderTokenLines.d.ts

@@ -0,0 +1,22 @@
+import { ElementContent } from "../node_modules/.pnpm/@types_hast@3.0.4/node_modules/@types/hast/index.js";
+import { DecorationItem, ThemedToken } from "../highlighter/types.js";
+import { SharedRenderState } from "../types.js";
+//#region src/utils/renderTokenLines.d.ts
+interface RenderTokenLinesOptions {
+  lazyLineAST?: boolean;
+  cacheHtmlStyles?: boolean;
+  state?: SharedRenderState;
+  useTokenTransformer?: boolean;
+  mergeWhitespaces?: 'never' | 'always';
+  decorations?: DecorationItem[];
+  /** UTF-16 line start offsets, required for numeric decoration positions. */
+  lineOffsets?: number[];
+}
+/**
+ * Converts highlighted tokens to one HAST element per line for custom renderers.
+ * Callers provide any line separators and enclosing pre/code elements.
+ */
+declare function renderTokenLines(lines: ThemedToken[][], { state, lazyLineAST, cacheHtmlStyles, useTokenTransformer, mergeWhitespaces, decorations, lineOffsets }: RenderTokenLinesOptions): ElementContent[];
+//#endregion
+export { RenderTokenLinesOptions, renderTokenLines };
+//# sourceMappingURL=renderTokenLines.d.ts.map

package/dist/utils/renderTokenLines.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"renderTokenLines.d.ts","names":[],"sources":["../../src/utils/renderTokenLines.ts"],"mappings":";;;;UAQiB;EACf;EACA;EACA,QAAQ;EACR;EACA;EACA,cAAc;;EAEd;;;;;;iBA0Bc,iBACd,OAAO,mBAEL,OACA,aACA,iBACA,qBACA,kBACA,aACA,eACC,0BACF"}

package/dist/utils/renderTokenLines.js

@@ -0,0 +1,250 @@
+import { processLine } from "./processLine.js";
+import { setDeferredArrayItem } from "./setDeferredArrayItem.js";
+import { tokenStyle } from "./tokenStyle.js";
+import { wrapTokenFragments } from "./wrapTokenFragments.js";
+//#region src/utils/renderTokenLines.ts
+/**
+* Converts highlighted tokens to one HAST element per line for custom renderers.
+* Callers provide any line separators and enclosing pre/code elements.
+*/
+function renderTokenLines(lines, { state, lazyLineAST = false, cacheHtmlStyles = false, useTokenTransformer = false, mergeWhitespaces = "always", decorations = [], lineOffsets = [] }) {
+	let decorationsByLine;
+	for (const decoration of decorations) {
+		const byLine = decorationsByLine ??= /* @__PURE__ */ new Map();
+		const normalized = {
+			...decoration,
+			start: getDecorationPosition(decoration.start, lineOffsets),
+			end: getDecorationPosition(decoration.end, lineOffsets)
+		};
+		const lastLine = Math.min(normalized.end.line, lines.length - 1);
+		for (let line = normalized.start.line; line <= lastLine; line++) {
+			let spans = byLine.get(line);
+			if (spans == null) byLine.set(line, spans = []);
+			spans.push(normalized);
+		}
+	}
+	const renderLine = (tokens, lineIndex) => {
+		let normalized;
+		if (useTokenTransformer || mergeWhitespaces !== "never") {
+			let pendingWhitespace = "";
+			let whitespaceOffset = 0;
+			for (let i = 0; i < tokens.length; i++) {
+				let token = tokens[i];
+				if (useTokenTransformer) {
+					const content = token.content.trim();
+					if (content !== "" && content !== token.content) {
+						normalized ??= tokens.slice(0, i);
+						const start = token.content.indexOf(content);
+						if (start > 0) normalized.push({
+							content: token.content.slice(0, start),
+							offset: token.offset
+						});
+						normalized.push({
+							...token,
+							content,
+							offset: token.offset + start
+						});
+						const end = start + content.length;
+						if (end < token.content.length) normalized.push({
+							content: token.content.slice(end),
+							offset: token.offset + end
+						});
+						continue;
+					}
+				} else {
+					const decoratedWhitespace = ((token.fontStyle ?? 0) & 12) !== 0;
+					if (!decoratedWhitespace && /^\s+$/.test(token.content) && i < tokens.length - 1) {
+						normalized ??= tokens.slice(0, i);
+						if (pendingWhitespace === "") whitespaceOffset = token.offset;
+						pendingWhitespace += token.content;
+						continue;
+					}
+					if (pendingWhitespace !== "" && normalized != null) {
+						if (decoratedWhitespace) normalized.push({
+							content: pendingWhitespace,
+							offset: whitespaceOffset
+						});
+						else token = {
+							...token,
+							content: pendingWhitespace + token.content,
+							offset: whitespaceOffset
+						};
+						pendingWhitespace = "";
+					}
+				}
+				normalized?.push(token);
+			}
+		}
+		normalized ??= tokens;
+		const line = {
+			type: "element",
+			tagName: state == null ? "span" : "div",
+			properties: state == null ? { class: "line" } : {},
+			children: []
+		};
+		const spans = decorationsByLine?.get(lineIndex);
+		const column = spans == null ? appendTokens(line, normalized, useTokenTransformer, cacheHtmlStyles) : appendDecoratedTokens(line, normalized, spans, lineIndex, useTokenTransformer, cacheHtmlStyles);
+		if (useTokenTransformer && column === 0) line.children.push({
+			type: "element",
+			tagName: "br",
+			properties: {},
+			children: []
+		});
+		if (useTokenTransformer && spans != null) wrapTokenFragments(line);
+		return state == null ? line : processLine(line, lineIndex + 1, state);
+	};
+	if (!lazyLineAST) return lines.map(renderLine);
+	const rows = new Array(lines.length);
+	for (let index = 0; index < lines.length; index++) {
+		const tokens = lines[index];
+		setDeferredArrayItem(rows, index, () => renderLine(tokens, index));
+	}
+	return rows;
+}
+function appendTokens(line, tokens, useTokenTransformer, cacheHtmlStyles) {
+	let column = 0;
+	for (const token of tokens) {
+		if (token.content === "") continue;
+		line.children.push(createTokenSpan(token, tokenStyle(token, cacheHtmlStyles), token.content, column, useTokenTransformer));
+		column += token.content.length;
+	}
+	return column;
+}
+function appendDecoratedTokens(line, tokens, spans, lineIndex, useTokenTransformer, cacheHtmlStyles) {
+	if (spans.length > 1) spans.sort((a, b) => {
+		if (a.start.line !== b.start.line) return a.start.line - b.start.line;
+		if (a.start.character !== b.start.character) return a.start.character - b.start.character;
+		if (a.end.line !== b.end.line) return b.end.line - a.end.line;
+		return b.end.character - a.end.character;
+	});
+	const lineLength = tokens.reduce((length, token) => length + token.content.length, 0);
+	const boundaries = [];
+	for (let order = 0; order < spans.length; order++) {
+		const decoration = spans[order];
+		const range = {
+			decoration,
+			order,
+			from: decoration.start.line === lineIndex ? decoration.start.character : 0,
+			to: decoration.end.line === lineIndex ? decoration.end.character : lineLength
+		};
+		boundaries.push({
+			position: Math.max(0, Math.min(range.from, lineLength)),
+			range,
+			start: true
+		}, {
+			position: Math.max(0, Math.min(range.to, lineLength)),
+			range,
+			start: false
+		});
+	}
+	boundaries.sort((a, b) => a.position - b.position);
+	const active = [];
+	const decorationStack = [];
+	let boundaryIndex = 0;
+	const updateActive = (range, add) => {
+		let low = 0;
+		let high = active.length;
+		while (low < high) {
+			const middle = low + high >>> 1;
+			if (active[middle].order < range.order) low = middle + 1;
+			else high = middle;
+		}
+		if (add) active.splice(low, 0, range);
+		else if (active[low] === range) active.splice(low, 1);
+	};
+	const getParent = () => {
+		let parent = line;
+		let depth = 0;
+		for (const range of active) {
+			if (decorationStack[depth]?.range !== range) {
+				decorationStack.length = depth;
+				const wrapper = {
+					type: "element",
+					tagName: "span",
+					properties: { ...range.decoration.properties },
+					children: []
+				};
+				parent.children.push(wrapper);
+				decorationStack.push({
+					range,
+					node: wrapper
+				});
+			}
+			parent = decorationStack[depth++].node;
+		}
+		decorationStack.length = depth;
+		return parent;
+	};
+	let parent = line;
+	const advanceBoundary = (position) => {
+		const first = boundaryIndex;
+		while (boundaries[boundaryIndex]?.position === position) boundaryIndex++;
+		let hasMarker = false;
+		for (let i = first; i < boundaryIndex; i++) {
+			const { range, start } = boundaries[i];
+			if (!start && range.from !== range.to && position < lineLength) updateActive(range, false);
+		}
+		for (let i = first; i < boundaryIndex; i++) {
+			const { range, start } = boundaries[i];
+			if (start && range.from <= position && range.to >= position && (range.to > position || range.from === position || position === lineLength)) {
+				updateActive(range, true);
+				if (range.from === range.to) hasMarker = true;
+			}
+		}
+		if (hasMarker) {
+			getParent();
+			for (let i = first; i < boundaryIndex; i++) {
+				const { range, start } = boundaries[i];
+				if (start && range.from === range.to) updateActive(range, false);
+			}
+		}
+		if (position < lineLength) parent = getParent();
+	};
+	let column = 0;
+	for (const token of tokens) {
+		if (token.content === "") continue;
+		const start = column;
+		const end = start + token.content.length;
+		const style = tokenStyle(token, cacheHtmlStyles);
+		while (column < end) {
+			if (boundaries[boundaryIndex]?.position === column) advanceBoundary(column);
+			const to = Math.min(end, boundaries[boundaryIndex]?.position ?? end);
+			parent.children.push(createTokenSpan(token, style, column === start && to === end ? token.content : token.content.slice(column - start, to - start), start, useTokenTransformer));
+			column = to;
+		}
+	}
+	if (boundaries[boundaryIndex]?.position === column) advanceBoundary(column);
+	return column;
+}
+function createTokenSpan(token, style, text, start, useTokenTransformer) {
+	const properties = token.htmlAttrs != null ? { ...token.htmlAttrs } : {};
+	if (style !== "") properties.style = style;
+	if (useTokenTransformer) properties["data-char"] = start;
+	return {
+		type: "element",
+		tagName: "span",
+		properties,
+		children: [{
+			type: "text",
+			value: text
+		}]
+	};
+}
+function getDecorationPosition(position, lineOffsets) {
+	if (typeof position !== "number") return position;
+	let low = 0;
+	let high = lineOffsets.length;
+	while (low + 1 < high) {
+		const middle = low + high >>> 1;
+		if (lineOffsets[middle] <= position) low = middle;
+		else high = middle;
+	}
+	return {
+		line: low,
+		character: position - (lineOffsets[low] ?? 0)
+	};
+}
+//#endregion
+export { renderTokenLines };
+
+//# sourceMappingURL=renderTokenLines.js.map