@pierre/diffs 1.5.2 → 1.6.0

package/dist/highlighter/shiki-editor.js

@@ -0,0 +1,483 @@
+import { debounce } from "../editor/utils.js";
+import { EncodedTokenMetadata, INITIAL } from "shiki/textmate";
+//#region src/highlighter/shiki-editor.ts
+const TOKENIZE_TIME_LIMIT = 0;
+let nextTokenizerId = 0;
+/** TextMate state caching and background work for a single editable document. */
+var ShikiEditorTokenizer = class {
+	backend;
+	#highlighter;
+	#grammar;
+	#themeName;
+	#colorMap;
+	#textDocument;
+	#tokenizeMaxLineLength;
+	#onDeferTokenize;
+	#matchBrackets;
+	#debug;
+	#isCleanedUp = false;
+	#stateStack = [INITIAL];
+	#comparisonStateStack = [];
+	#comparisonStateStackStart = 0;
+	#comparisonLineChanges = [];
+	#lastLine = -1;
+	#isStopped = true;
+	#isPaused = false;
+	#backgroundJobId = 0;
+	#tokenizerId = ++nextTokenizerId;
+	#backgroundPrebuildEndLine = -1;
+	#pendingPrebuildEndLine = -1;
+	#backgroundChangedLineRanges;
+	#backgroundChangedRangeIndex = 0;
+	#bracketIgnoredRanges = [];
+	#isMessageListenerAttached = false;
+	#prebuildStateStack = debounce(async (renderRange) => {
+		if (this.#isCleanedUp) return;
+		const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};
+		const endLine = Math.min(totalLines === Infinity ? Infinity : startingLine + totalLines, this.#textDocument.lineCount);
+		if (this.#grammar == null && !isGrammarlessLanguage(this.#textDocument.languageId)) {
+			await this.backend?.loadLanguages([this.#textDocument.languageId]);
+			if (this.#isCleanedUp) return;
+			this.#grammar = this.#highlighter.getLanguage(this.#textDocument.languageId);
+		}
+		this.#ensureActiveTheme();
+		this.#scheduleStatePrebuild(endLine);
+	}, 500);
+	#onMessage = ({ data }) => {
+		if (typeof data !== "object" || data === null) return;
+		const { type, tokenizerId, jobId } = data;
+		if (type === "tokenize" && tokenizerId === this.#tokenizerId && typeof jobId === "number" && jobId === this.#backgroundJobId) {
+			if (this.#backgroundPrebuildEndLine >= 0) this.#backgroundPrebuild(jobId);
+			else this.#backgroundTokenize(jobId);
+		}
+	};
+	getStringCommentRegexpRangesInLine(lineIndex) {
+		if (!this.#matchBrackets || lineIndex < 0 || lineIndex >= this.#textDocument.lineCount) return null;
+		this.#ensureGrammar();
+		if (this.#grammar == null) return null;
+		if (this.#bracketIgnoredRanges[lineIndex] === void 0) {
+			this.#buildStateStack(lineIndex);
+			const state = this.#stateStack[lineIndex] ?? INITIAL;
+			const result = this.#tokenizeLineAt(lineIndex, state);
+			this.#stateStack[lineIndex + 1] = result.state;
+		}
+		return this.#bracketIgnoredRanges[lineIndex] ?? null;
+	}
+	constructor(highlighter, options, backend) {
+		this.backend = backend;
+		this.#highlighter = highlighter;
+		this.#textDocument = options.textDocument;
+		this.#themeName = options.theme;
+		this.#colorMap = [];
+		this.#tokenizeMaxLineLength = options.tokenizeMaxLineLength ?? 1e3;
+		this.#onDeferTokenize = options.onDeferTokenize;
+		this.#matchBrackets = options.matchBrackets !== false;
+		this.#debug = options.__debug ?? false;
+		this.#ensureGrammar();
+		this.#ensureActiveTheme();
+	}
+	setTheme(themeName) {
+		if (this.#themeName === themeName) return;
+		this.#themeName = themeName;
+		this.#ensureActiveTheme();
+		this.stopBackgroundTokenize();
+		this.#stateStack = [INITIAL];
+		this.#comparisonStateStack = [];
+		this.#comparisonStateStackStart = 0;
+		this.#comparisonLineChanges = [];
+		if (this.#grammar != null && this.#textDocument.lineCount > 0) this.#scheduleBackgroundTokenize(0);
+	}
+	#ensureActiveTheme() {
+		if (this.#themeName === "") return;
+		this.backend?.getTheme(this.#themeName);
+		const { colorMap } = this.#highlighter.setTheme(this.#themeName);
+		this.#colorMap = colorMap;
+	}
+	dispose() {
+		this.#isCleanedUp = true;
+		this.stopBackgroundTokenize();
+		this.#detachMessageListener();
+	}
+	tokenize(change, renderRange, hostRealignsRows = false) {
+		this.#ensureGrammar();
+		this.#ensureActiveTheme();
+		if (this.#grammar == null && !isGrammarlessLanguage(this.#textDocument.languageId)) throw new Error(`Grammar for language "${this.#textDocument.languageId}" not loaded`);
+		const { lineCount } = this.#textDocument;
+		const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};
+		const renderRangeEndLine = totalLines === Infinity ? lineCount : Math.min(startingLine + totalLines, lineCount);
+		const dirtyStart = change.startLine;
+		const viewStart = Math.max(startingLine, dirtyStart);
+		const crossesRenderRangeEnd = renderRange != null && totalLines !== Infinity && change.lineDelta > 0 && dirtyStart < renderRangeEndLine && change.endLine >= renderRangeEndLine;
+		const canReuseCachedStates = change.lineDelta === 0 && (change.changedLineChanges?.every(([, , lineDelta]) => lineDelta === 0) ?? true);
+		if (this.#matchBrackets && !canReuseCachedStates) this.#bracketIgnoredRanges.length = Math.min(this.#bracketIgnoredRanges.length, change.startLine);
+		const canReuseShiftedStates = hostRealignsRows && change.lineDelta !== 0 && dirtyStart >= startingLine;
+		const canCacheTokenizedStates = canReuseCachedStates || renderRange == null || dirtyStart >= viewStart;
+		const changedLineRanges = change.changedLineRanges ?? [[dirtyStart, change.endLine]];
+		this.#comparisonStateStack = [];
+		this.#comparisonStateStackStart = 0;
+		this.#comparisonLineChanges = [];
+		let offscreenSyncEnd = -1;
+		if (dirtyStart < viewStart) {
+			for (const [rangeStart, rangeEnd] of changedLineRanges) if (rangeStart < viewStart) offscreenSyncEnd = Math.max(offscreenSyncEnd, Math.min(rangeEnd, viewStart - 1));
+		}
+		const shouldFlushOffscreenLines = offscreenSyncEnd >= dirtyStart && (canReuseCachedStates || change.lineDelta < 0);
+		if (canReuseCachedStates) this.#buildStateStack(dirtyStart);
+		else {
+			this.#shiftComparisonStateStack(change);
+			if (renderRange == null || dirtyStart >= viewStart) this.#buildStateStack(viewStart);
+		}
+		let changedRangeIndex = 0;
+		let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];
+		let backgroundStartLine;
+		let backgroundChangedRangeIndex = 0;
+		let line = canReuseCachedStates ? changedLineRanges[changedRangeIndex][0] : viewStart;
+		let settled = false;
+		const dirtyLines = /* @__PURE__ */ new Map();
+		const offscreenDirtyLines = shouldFlushOffscreenLines ? /* @__PURE__ */ new Map() : void 0;
+		if (offscreenDirtyLines != null && !canReuseCachedStates) {
+			const offscreenEnd = Math.min(offscreenSyncEnd + 1, viewStart, renderRangeEndLine);
+			if (offscreenEnd > dirtyStart) {
+				this.#buildStateStack(offscreenEnd);
+				let offscreenLine = dirtyStart;
+				let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;
+				for (; offscreenLine < offscreenEnd; offscreenLine++) {
+					const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);
+					offscreenState = resolved.state;
+					offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);
+				}
+				this.#stateStack[offscreenEnd] = offscreenState;
+			}
+		}
+		let state = this.#stateStack[line] ?? INITIAL;
+		for (; line < renderRangeEndLine;) {
+			const previousNextState = canReuseCachedStates ? this.#stateStack[line + 1] : canReuseShiftedStates ? this.#getPreviousEndState(line + 1) : void 0;
+			if (canCacheTokenizedStates) this.#stateStack[line] = state;
+			const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(line, state);
+			state = nextState;
+			if (line >= viewStart) dirtyLines.set(line, resolvedTokens);
+			else offscreenDirtyLines?.set(line, resolvedTokens);
+			if (canCacheTokenizedStates) this.#stateStack[line + 1] = state;
+			settled = line >= currentChangedRangeEnd && (canReuseCachedStates || canReuseShiftedStates) && previousNextState != null && state.equals(previousNextState);
+			if (settled) {
+				changedRangeIndex++;
+				const nextRange = changedLineRanges[changedRangeIndex];
+				if (nextRange == null) break;
+				if (nextRange[0] >= renderRangeEndLine) {
+					backgroundStartLine = nextRange[0];
+					backgroundChangedRangeIndex = changedRangeIndex;
+					break;
+				}
+				let nextState = this.#stateStack[nextRange[0]];
+				if (canReuseShiftedStates) for (let stateLine = line + 2; stateLine <= nextRange[0]; stateLine++) {
+					nextState = this.#getPreviousEndState(stateLine);
+					if (nextState == null) break;
+					this.#stateStack[stateLine] = nextState;
+				}
+				if (nextState == null) {
+					currentChangedRangeEnd = nextRange[1];
+					line++;
+				} else {
+					line = nextRange[0];
+					state = nextState;
+					currentChangedRangeEnd = nextRange[1];
+				}
+				settled = false;
+				continue;
+			}
+			line++;
+		}
+		if (canCacheTokenizedStates) {
+			if (line < renderRangeEndLine) this.#stateStack[line + 1] = state;
+			else this.#stateStack[line] = state;
+		}
+		if (settled && canReuseShiftedStates && backgroundStartLine == null) {
+			for (let stateLine = line + 2; stateLine <= lineCount; stateLine++) {
+				const previousState = this.#getPreviousEndState(stateLine);
+				if (previousState == null) break;
+				this.#stateStack[stateLine] = previousState;
+			}
+			this.#comparisonStateStack = [];
+			this.#comparisonStateStackStart = 0;
+			this.#comparisonLineChanges = [];
+		}
+		if (offscreenDirtyLines != null && offscreenDirtyLines.size > 0) this.#onDeferTokenize(offscreenDirtyLines);
+		if (backgroundStartLine != null) {
+			if (this.#matchBrackets && canReuseCachedStates) this.#bracketIgnoredRanges.length = Math.min(this.#bracketIgnoredRanges.length, backgroundStartLine);
+			this.#scheduleBackgroundTokenize(backgroundStartLine, changedLineRanges, backgroundChangedRangeIndex);
+		} else if (!settled && line < lineCount) {
+			const backgroundLine = crossesRenderRangeEnd && dirtyStart >= viewStart ? renderRangeEndLine : dirtyStart < viewStart && !canReuseCachedStates ? dirtyStart : line;
+			if (this.#matchBrackets && canReuseCachedStates) this.#bracketIgnoredRanges.length = Math.min(this.#bracketIgnoredRanges.length, backgroundLine);
+			this.#scheduleBackgroundTokenize(backgroundLine, changedLineRanges, changedRangeIndex);
+		}
+		return dirtyLines;
+	}
+	prebuildStateStack(renderRange) {
+		this.#ensureGrammar();
+		this.#prebuildStateStack(renderRange);
+	}
+	stopBackgroundTokenize() {
+		this.#pendingPrebuildEndLine = -1;
+		if (this.#isStopped) return;
+		this.#isStopped = true;
+		this.#isPaused = false;
+		this.#lastLine = -1;
+		this.#backgroundPrebuildEndLine = -1;
+		this.#backgroundChangedLineRanges = void 0;
+		this.#backgroundChangedRangeIndex = 0;
+		this.#comparisonStateStack = [];
+		this.#comparisonStateStackStart = 0;
+		this.#comparisonLineChanges = [];
+		this.#detachMessageListener();
+	}
+	pauseBackgroundTokenize() {
+		if (this.#isStopped || this.#isPaused) return;
+		if (this.#debug) console.log("[diffs/editor] background tokenization paused", { jobId: this.#backgroundJobId });
+		this.#isPaused = true;
+	}
+	resumeBackgroundTokenize() {
+		if (this.#isStopped || !this.#isPaused || this.#grammar == null || this.#lastLine < 0) return;
+		if (this.#debug) console.log("[diffs/editor] background tokenization resumed", { jobId: this.#backgroundJobId });
+		this.#isPaused = false;
+		this.#postTokenizeMessage(this.#backgroundJobId);
+	}
+	#ensureGrammar() {
+		if (this.#grammar == null && !isGrammarlessLanguage(this.#textDocument.languageId) && this.#highlighter.getLoadedLanguages().includes(this.#textDocument.languageId)) this.#grammar = this.#highlighter.getLanguage(this.#textDocument.languageId);
+	}
+	#attachMessageListener() {
+		if (this.#isMessageListenerAttached) return;
+		globalThis.addEventListener("message", this.#onMessage);
+		this.#isMessageListenerAttached = true;
+	}
+	#detachMessageListener() {
+		if (!this.#isMessageListenerAttached) return;
+		globalThis.removeEventListener("message", this.#onMessage);
+		this.#isMessageListenerAttached = false;
+	}
+	#postTokenizeMessage(jobId) {
+		globalThis.postMessage({
+			type: "tokenize",
+			tokenizerId: this.#tokenizerId,
+			jobId
+		});
+	}
+	#scheduleBackgroundTokenize(startLine, changedLineRanges, changedRangeIndex = 0) {
+		if (isGrammarlessLanguage(this.#textDocument.languageId)) return;
+		const jobId = ++this.#backgroundJobId;
+		if (this.#debug) console.log("[diffs/editor] background tokenization scheduled", {
+			jobId,
+			startLine,
+			changedLineRanges,
+			changedRangeIndex
+		});
+		this.#isStopped = false;
+		this.#isPaused = false;
+		this.#lastLine = startLine;
+		if (this.#backgroundPrebuildEndLine >= 0) this.#pendingPrebuildEndLine = Math.max(this.#pendingPrebuildEndLine, this.#backgroundPrebuildEndLine);
+		this.#backgroundPrebuildEndLine = -1;
+		this.#backgroundChangedLineRanges = changedLineRanges;
+		this.#backgroundChangedRangeIndex = changedRangeIndex;
+		this.#attachMessageListener();
+		this.#postTokenizeMessage(jobId);
+	}
+	#scheduleStatePrebuild(endLine) {
+		if (this.#grammar == null || this.#stateStack.length > endLine) return;
+		if (!this.#isStopped) {
+			if (this.#backgroundPrebuildEndLine >= 0) this.#backgroundPrebuildEndLine = Math.max(this.#backgroundPrebuildEndLine, endLine);
+			else this.#pendingPrebuildEndLine = Math.max(this.#pendingPrebuildEndLine, endLine);
+			return;
+		}
+		const jobId = ++this.#backgroundJobId;
+		this.#isStopped = false;
+		this.#isPaused = false;
+		this.#lastLine = this.#stateStack.length - 1;
+		this.#backgroundPrebuildEndLine = endLine;
+		this.#pendingPrebuildEndLine = -1;
+		this.#backgroundChangedLineRanges = void 0;
+		this.#backgroundChangedRangeIndex = 0;
+		this.#attachMessageListener();
+		this.#postTokenizeMessage(jobId);
+	}
+	#tokenizeLineAt(line, state) {
+		const lineText = this.#textDocument.getLineText(line);
+		if (lineText.length >= this.#tokenizeMaxLineLength) {
+			console.warn(`[diffs] Line(${line}) too long to tokenize: ${lineText.length}`);
+			this.#cacheBracketIgnoredRanges(line, null);
+			return {
+				resolvedTokens: [[
+					0,
+					"",
+					lineText
+				]],
+				state
+			};
+		}
+		if (this.#grammar == null || lineText === "" || lineText.trim() === "") {
+			this.#cacheBracketIgnoredRanges(line, null);
+			return {
+				resolvedTokens: [[
+					0,
+					"",
+					lineText
+				]],
+				state
+			};
+		}
+		const result = tokenizeLine(this.#grammar, this.#colorMap, lineText, state, TOKENIZE_TIME_LIMIT, this.#matchBrackets);
+		this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges);
+		return {
+			resolvedTokens: result.resolvedTokens,
+			state: result.ruleStack
+		};
+	}
+	#cacheBracketIgnoredRanges(line, ranges) {
+		if (this.#matchBrackets) this.#bracketIgnoredRanges[line] = ranges;
+	}
+	#shiftComparisonStateStack(change) {
+		const lineChanges = change.changedLineChanges ?? [[
+			change.startLine,
+			change.endLine,
+			change.lineDelta
+		]];
+		const comparisonStart = change.startLine + 1;
+		if (comparisonStart <= this.#stateStack.length - comparisonStart) {
+			this.#comparisonStateStack = this.#stateStack;
+			this.#comparisonStateStackStart = 0;
+			this.#stateStack = this.#stateStack.slice(0, comparisonStart);
+		} else {
+			this.#comparisonStateStackStart = comparisonStart;
+			this.#comparisonStateStack = this.#stateStack.slice(comparisonStart);
+			this.#stateStack.length = Math.min(this.#stateStack.length, comparisonStart);
+		}
+		this.#comparisonLineChanges = lineChanges;
+	}
+	#getPreviousEndState(line) {
+		let previousLine = line;
+		for (let index = this.#comparisonLineChanges.length - 1; index >= 0; index--) {
+			const [startLine, endLine, lineDelta] = this.#comparisonLineChanges[index];
+			if (lineDelta === 0) continue;
+			if (previousLine > endLine) previousLine -= lineDelta;
+			else if (previousLine > startLine) return this.#stateStack[line];
+		}
+		return this.#comparisonStateStack[previousLine - this.#comparisonStateStackStart] ?? this.#stateStack[line];
+	}
+	#buildStateStack(endAt, timeBudget) {
+		const boundedEndAt = Math.min(Math.max(0, endAt), this.#textDocument.lineCount);
+		if (this.#stateStack.length > boundedEndAt || this.#grammar == null) return true;
+		const startedAt = timeBudget == null ? 0 : performance.now();
+		let line = this.#stateStack.length - 1;
+		let state = this.#stateStack[line] ?? INITIAL;
+		while (line < boundedEndAt) {
+			this.#stateStack[line] = state;
+			const lineText = this.#textDocument.getLineText(line);
+			if (lineText.length < this.#tokenizeMaxLineLength && lineText !== "" && lineText.trim() !== "") {
+				const result = tokenizeLine(this.#grammar, this.#colorMap, lineText, state, TOKENIZE_TIME_LIMIT, this.#matchBrackets, false);
+				this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges);
+				state = result.ruleStack;
+			} else this.#cacheBracketIgnoredRanges(line, null);
+			line++;
+			this.#stateStack[line] = state;
+			if (timeBudget != null && performance.now() - startedAt > timeBudget) break;
+		}
+		return line >= boundedEndAt;
+	}
+	#backgroundPrebuild(jobId) {
+		if (this.#isStopped || this.#isPaused || this.#grammar == null || jobId !== this.#backgroundJobId) return;
+		this.#ensureActiveTheme();
+		const complete = this.#buildStateStack(this.#backgroundPrebuildEndLine, 1);
+		if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) return;
+		if (complete) {
+			this.stopBackgroundTokenize();
+			return;
+		}
+		this.#lastLine = this.#stateStack.length - 1;
+		this.#postTokenizeMessage(jobId);
+	}
+	#backgroundTokenize(jobId) {
+		if (this.#isStopped || this.#isPaused || this.#grammar == null || jobId !== this.#backgroundJobId) return;
+		this.#ensureActiveTheme();
+		const t = performance.now();
+		const lines = /* @__PURE__ */ new Map();
+		const totalLines = this.#textDocument.lineCount;
+		const changedLineRanges = this.#backgroundChangedLineRanges;
+		let line = this.#lastLine;
+		let state = this.#stateStack[line] ?? INITIAL;
+		let settled = false;
+		let changedRangeIndex = this.#backgroundChangedRangeIndex;
+		let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];
+		for (; line < totalLines;) {
+			this.#stateStack[line] = state;
+			const previousNextState = currentChangedRangeEnd != null ? this.#getPreviousEndState(line + 1) : void 0;
+			const result = this.#tokenizeLineAt(line, state);
+			lines.set(line, result.resolvedTokens);
+			state = result.state;
+			this.#stateStack[line + 1] = state;
+			settled = currentChangedRangeEnd != null && line >= currentChangedRangeEnd && previousNextState != null && state.equals(previousNextState);
+			line++;
+			if (settled) {
+				changedRangeIndex++;
+				const nextRange = changedLineRanges?.[changedRangeIndex];
+				if (nextRange == null) break;
+				currentChangedRangeEnd = nextRange[1];
+				if (this.#stateStack[nextRange[0]] == null) settled = false;
+				else {
+					line = nextRange[0];
+					state = this.#stateStack[line] ?? state;
+					settled = false;
+					continue;
+				}
+			}
+			if (performance.now() - t > 1) break;
+		}
+		this.#onDeferTokenize(lines);
+		if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) return;
+		if (settled || line >= totalLines) {
+			const pendingPrebuildEndLine = this.#pendingPrebuildEndLine;
+			this.stopBackgroundTokenize();
+			if (pendingPrebuildEndLine >= 0) this.#scheduleStatePrebuild(pendingPrebuildEndLine);
+			return;
+		}
+		this.#lastLine = line;
+		this.#backgroundChangedRangeIndex = changedRangeIndex;
+		this.#postTokenizeMessage(jobId);
+	}
+};
+function tokenizeLine(grammar, colorMap, lineText, stateStack, timeLimit, collectBracketIgnoredRanges = true, resolveTokens = true) {
+	const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);
+	if (result.stoppedEarly) console.warn(`[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`);
+	const rawTokens = result.tokens;
+	const tokensLength = rawTokens.length / 2;
+	const resolvedTokens = [];
+	const bracketIgnoredRanges = [];
+	if (!resolveTokens && !collectBracketIgnoredRanges) return {
+		ruleStack: result.ruleStack,
+		resolvedTokens,
+		bracketIgnoredRanges
+	};
+	for (let j = 0; j < tokensLength; j++) {
+		const offset = rawTokens[2 * j];
+		const nextOffset = j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;
+		if (offset === nextOffset) continue;
+		const metadata = rawTokens[2 * j + 1];
+		if (resolveTokens) {
+			const fg = EncodedTokenMetadata.getForeground(metadata);
+			resolvedTokens.push([
+				offset,
+				colorMap[fg],
+				lineText.slice(offset, nextOffset)
+			]);
+		}
+		if (collectBracketIgnoredRanges && EncodedTokenMetadata.getTokenType(metadata) > 0) bracketIgnoredRanges.push([offset, nextOffset]);
+	}
+	return {
+		ruleStack: result.ruleStack,
+		resolvedTokens,
+		bracketIgnoredRanges
+	};
+}
+function isGrammarlessLanguage(languageId) {
+	return languageId === "text" || languageId === "ansi";
+}
+//#endregion
+export { ShikiEditorTokenizer };
+
+//# sourceMappingURL=shiki-editor.js.map

package/dist/highlighter/shiki-editor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shiki-editor.js","names":[],"sources":["../../src/highlighter/shiki-editor.ts"],"sourcesContent":["import type { HighlighterCore } from 'shiki/core';\nimport {\n  EncodedTokenMetadata,\n  type IGrammar,\n  INITIAL,\n  type StateStack,\n} from 'shiki/textmate';\n\nimport type { TextDocumentChange } from '../editor/textDocument';\nimport { debounce } from '../editor/utils';\nimport type { HighlightedToken, RenderRange } from '../types';\nimport type { DiffsEditorTokenizer } from './DiffsEditorTokenizer';\nimport type { DiffsEditorTokenizerOptions } from './tokenizer-types';\nimport type { DiffsHighlighter } from './types';\n\n// Timed aborts can leave incomplete grammar state; limit line length instead.\nconst TOKENIZE_TIME_LIMIT = 0;\nlet nextTokenizerId = 0;\n\n/** TextMate state caching and background work for a single editable document. */\nexport class ShikiEditorTokenizer implements DiffsEditorTokenizer {\n  #highlighter: HighlighterCore;\n  #grammar: IGrammar | undefined;\n  #themeName: string;\n  #colorMap: string[];\n  #textDocument: DiffsEditorTokenizerOptions['textDocument'];\n  #tokenizeMaxLineLength: number;\n  #onDeferTokenize: DiffsEditorTokenizerOptions['onDeferTokenize'];\n  #matchBrackets: boolean;\n  #debug: boolean;\n  #isCleanedUp = false;\n  #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index\n  #comparisonStateStack: StateStack[] = [];\n  #comparisonStateStackStart = 0;\n  #comparisonLineChanges: NonNullable<\n    TextDocumentChange['changedLineChanges']\n  > = [];\n  #lastLine: number = -1;\n  #isStopped: boolean = true;\n  #isPaused: boolean = false;\n  #backgroundJobId: number = 0;\n  #tokenizerId = ++nextTokenizerId;\n  #backgroundPrebuildEndLine = -1;\n  #pendingPrebuildEndLine = -1;\n  #backgroundChangedLineRanges: readonly [number, number][] | undefined;\n  #backgroundChangedRangeIndex: number = 0;\n  #bracketIgnoredRanges: ([number, number][] | null | undefined)[] = [];\n  #isMessageListenerAttached: boolean = false;\n\n  #prebuildStateStack = debounce(async (renderRange?: RenderRange) => {\n    // A queued timer may run after the highlighter is disposed.\n    if (this.#isCleanedUp) {\n      return;\n    }\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const endLine = Math.min(\n      totalLines === Infinity ? Infinity : startingLine + totalLines,\n      this.#textDocument.lineCount\n    );\n    if (\n      this.#grammar == null &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      await this.backend?.loadLanguages([this.#textDocument.languageId]);\n      if (this.#isCleanedUp) {\n        return;\n      }\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n    this.#ensureActiveTheme();\n    this.#scheduleStatePrebuild(endLine);\n  }, 500);\n\n  #onMessage = ({ data }: MessageEvent<unknown>) => {\n    if (typeof data !== 'object' || data === null) {\n      return;\n    }\n    const { type, tokenizerId, jobId } = data as {\n      type?: unknown;\n      tokenizerId?: unknown;\n      jobId?: unknown;\n    };\n    if (\n      type === 'tokenize' &&\n      tokenizerId === this.#tokenizerId &&\n      typeof jobId === 'number' &&\n      jobId === this.#backgroundJobId\n    ) {\n      if (this.#backgroundPrebuildEndLine >= 0) {\n        this.#backgroundPrebuild(jobId);\n      } else {\n        this.#backgroundTokenize(jobId);\n      }\n    }\n  };\n\n  getStringCommentRegexpRangesInLine(\n    lineIndex: number\n  ): [number, number][] | null {\n    if (\n      !this.#matchBrackets ||\n      lineIndex < 0 ||\n      lineIndex >= this.#textDocument.lineCount\n    ) {\n      return null;\n    }\n    this.#ensureGrammar();\n    if (this.#grammar == null) {\n      return null;\n    }\n    if (this.#bracketIgnoredRanges[lineIndex] === undefined) {\n      this.#buildStateStack(lineIndex);\n      const state = this.#stateStack[lineIndex] ?? INITIAL;\n      const result = this.#tokenizeLineAt(lineIndex, state);\n      this.#stateStack[lineIndex + 1] = result.state;\n    }\n    return this.#bracketIgnoredRanges[lineIndex] ?? null;\n  }\n\n  constructor(\n    highlighter: HighlighterCore,\n    options: DiffsEditorTokenizerOptions,\n    private readonly backend?: Pick<\n      DiffsHighlighter,\n      'getTheme' | 'loadLanguages'\n    >\n  ) {\n    this.#highlighter = highlighter;\n    this.#textDocument = options.textDocument;\n    this.#themeName = options.theme;\n    this.#colorMap = [];\n    // The cutoff must match codeToTokens and Highlights: length >= limit.\n    this.#tokenizeMaxLineLength = options.tokenizeMaxLineLength ?? 1000;\n    this.#onDeferTokenize = options.onDeferTokenize;\n    this.#matchBrackets = options.matchBrackets !== false;\n    this.#debug = options.__debug ?? false;\n    this.#ensureGrammar();\n    this.#ensureActiveTheme();\n  }\n\n  setTheme(themeName: string): void {\n    if (this.#themeName === themeName) return;\n    this.#themeName = themeName;\n    this.#ensureActiveTheme();\n    this.stopBackgroundTokenize();\n    this.#stateStack = [INITIAL];\n    this.#comparisonStateStack = [];\n    this.#comparisonStateStackStart = 0;\n    this.#comparisonLineChanges = [];\n    if (this.#grammar != null && this.#textDocument.lineCount > 0) {\n      this.#scheduleBackgroundTokenize(0);\n    }\n  }\n\n  // Dual-theme SSR can change the shared highlighter's active theme. Restore\n  // the editor theme before tokenizing so color indices match its color map.\n  #ensureActiveTheme(): void {\n    if (this.#themeName === '') {\n      return;\n    }\n    // Resolving a theme does not attach it to Shiki until the backend uses it.\n    this.backend?.getTheme(this.#themeName);\n    const { colorMap } = this.#highlighter.setTheme(this.#themeName);\n    this.#colorMap = colorMap;\n  }\n\n  dispose(): void {\n    this.#isCleanedUp = true;\n    this.stopBackgroundTokenize();\n    this.#detachMessageListener();\n  }\n\n  // Prebuild viewport state with prebuildStateStack before the first edit.\n  tokenize(\n    change: TextDocumentChange,\n    renderRange?: RenderRange,\n    hostRealignsRows = false\n  ): Map<number, Array<HighlightedToken>> {\n    this.#ensureGrammar();\n    this.#ensureActiveTheme();\n    if (\n      this.#grammar == null &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      throw new Error(\n        `Grammar for language \"${this.#textDocument.languageId}\" not loaded`\n      );\n    }\n\n    const { lineCount } = this.#textDocument;\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const renderRangeEndLine =\n      totalLines === Infinity\n        ? lineCount\n        : Math.min(startingLine + totalLines, lineCount);\n\n    const dirtyStart = change.startLine;\n    const viewStart = Math.max(startingLine, dirtyStart);\n    const crossesRenderRangeEnd =\n      renderRange != null &&\n      totalLines !== Infinity &&\n      change.lineDelta > 0 &&\n      dirtyStart < renderRangeEndLine &&\n      change.endLine >= renderRangeEndLine;\n    const canReuseCachedStates =\n      change.lineDelta === 0 &&\n      (change.changedLineChanges?.every(([, , lineDelta]) => lineDelta === 0) ??\n        true);\n    if (this.#matchBrackets && !canReuseCachedStates) {\n      // Structural edits shift line indexes; only the unchanged prefix is valid.\n      this.#bracketIgnoredRanges.length = Math.min(\n        this.#bracketIgnoredRanges.length,\n        change.startLine\n      );\n    }\n    const canReuseShiftedStates =\n      hostRealignsRows && change.lineDelta !== 0 && dirtyStart >= startingLine;\n    const canCacheTokenizedStates =\n      canReuseCachedStates || renderRange == null || dirtyStart >= viewStart;\n    const changedLineRanges: readonly [number, number][] =\n      change.changedLineRanges ?? [[dirtyStart, change.endLine]];\n    this.#comparisonStateStack = [];\n    this.#comparisonStateStackStart = 0;\n    this.#comparisonLineChanges = [];\n    let offscreenSyncEnd = -1;\n    if (dirtyStart < viewStart) {\n      for (const [rangeStart, rangeEnd] of changedLineRanges) {\n        if (rangeStart < viewStart) {\n          offscreenSyncEnd = Math.max(\n            offscreenSyncEnd,\n            Math.min(rangeEnd, viewStart - 1)\n          );\n        }\n      }\n    }\n    const shouldFlushOffscreenLines =\n      offscreenSyncEnd >= dirtyStart &&\n      (canReuseCachedStates || change.lineDelta < 0);\n    if (canReuseCachedStates) {\n      this.#buildStateStack(dirtyStart);\n    } else {\n      this.#shiftComparisonStateStack(change);\n      if (renderRange == null || dirtyStart >= viewStart) {\n        this.#buildStateStack(viewStart);\n      }\n    }\n\n    let changedRangeIndex = 0;\n    let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];\n    let backgroundStartLine: number | undefined;\n    let backgroundChangedRangeIndex = 0;\n    let line = canReuseCachedStates\n      ? changedLineRanges[changedRangeIndex][0]\n      : viewStart;\n    let settled = false;\n    const dirtyLines: Map<number, Array<HighlightedToken>> = new Map();\n    const offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined = shouldFlushOffscreenLines ? new Map() : undefined;\n    if (offscreenDirtyLines != null && !canReuseCachedStates) {\n      const offscreenEnd = Math.min(\n        offscreenSyncEnd + 1,\n        viewStart,\n        renderRangeEndLine\n      );\n      if (offscreenEnd > dirtyStart) {\n        this.#buildStateStack(offscreenEnd);\n        let offscreenLine = dirtyStart;\n        let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;\n        for (; offscreenLine < offscreenEnd; offscreenLine++) {\n          const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);\n          offscreenState = resolved.state;\n          offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);\n        }\n        this.#stateStack[offscreenEnd] = offscreenState;\n      }\n    }\n    // Read state after the offscreen flush; a deletion may have invalidated\n    // the viewport's starting state inside a block comment or template literal.\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < renderRangeEndLine; ) {\n      const previousNextState = canReuseCachedStates\n        ? this.#stateStack[line + 1]\n        : canReuseShiftedStates\n          ? this.#getPreviousEndState(line + 1)\n          : undefined;\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line] = state;\n      }\n\n      const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(\n        line,\n        state\n      );\n      state = nextState;\n\n      if (line >= viewStart) {\n        dirtyLines.set(line, resolvedTokens);\n      } else {\n        offscreenDirtyLines?.set(line, resolvedTokens);\n      }\n\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line + 1] = state;\n      }\n      settled =\n        line >= currentChangedRangeEnd &&\n        (canReuseCachedStates || canReuseShiftedStates) &&\n        previousNextState != null &&\n        state.equals(previousNextState);\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges[changedRangeIndex];\n        if (nextRange == null) {\n          break;\n        }\n        if (nextRange[0] >= renderRangeEndLine) {\n          backgroundStartLine = nextRange[0];\n          backgroundChangedRangeIndex = changedRangeIndex;\n          break;\n        }\n        let nextState: StateStack | undefined = this.#stateStack[nextRange[0]];\n        if (canReuseShiftedStates) {\n          for (\n            let stateLine = line + 2;\n            stateLine <= nextRange[0];\n            stateLine++\n          ) {\n            nextState = this.#getPreviousEndState(stateLine);\n            if (nextState == null) {\n              break;\n            }\n            this.#stateStack[stateLine] = nextState;\n          }\n        }\n        if (nextState == null) {\n          currentChangedRangeEnd = nextRange[1];\n          line++;\n        } else {\n          line = nextRange[0];\n          state = nextState;\n          currentChangedRangeEnd = nextRange[1];\n        }\n        settled = false;\n        continue;\n      }\n      line++;\n    }\n\n    if (canCacheTokenizedStates) {\n      if (line < renderRangeEndLine) {\n        this.#stateStack[line + 1] = state;\n      } else {\n        this.#stateStack[line] = state;\n      }\n    }\n\n    if (settled && canReuseShiftedStates && backgroundStartLine == null) {\n      for (let stateLine = line + 2; stateLine <= lineCount; stateLine++) {\n        const previousState = this.#getPreviousEndState(stateLine);\n        if (previousState == null) {\n          break;\n        }\n        this.#stateStack[stateLine] = previousState;\n      }\n      this.#comparisonStateStack = [];\n      this.#comparisonStateStackStart = 0;\n      this.#comparisonLineChanges = [];\n    }\n\n    if (offscreenDirtyLines != null && offscreenDirtyLines.size > 0) {\n      this.#onDeferTokenize(offscreenDirtyLines);\n    }\n\n    if (backgroundStartLine != null) {\n      if (this.#matchBrackets && canReuseCachedStates) {\n        this.#bracketIgnoredRanges.length = Math.min(\n          this.#bracketIgnoredRanges.length,\n          backgroundStartLine\n        );\n      }\n      this.#scheduleBackgroundTokenize(\n        backgroundStartLine,\n        changedLineRanges,\n        backgroundChangedRangeIndex\n      );\n    } else if (!settled && line < lineCount) {\n      const backgroundLine =\n        crossesRenderRangeEnd && dirtyStart >= viewStart\n          ? renderRangeEndLine\n          : dirtyStart < viewStart && !canReuseCachedStates\n            ? dirtyStart\n            : line;\n      if (this.#matchBrackets && canReuseCachedStates) {\n        this.#bracketIgnoredRanges.length = Math.min(\n          this.#bracketIgnoredRanges.length,\n          backgroundLine\n        );\n      }\n      this.#scheduleBackgroundTokenize(\n        backgroundLine,\n        changedLineRanges,\n        changedRangeIndex\n      );\n    }\n\n    return dirtyLines;\n  }\n\n  prebuildStateStack(renderRange?: RenderRange): void {\n    this.#ensureGrammar();\n    this.#prebuildStateStack(renderRange);\n  }\n\n  stopBackgroundTokenize(): void {\n    this.#pendingPrebuildEndLine = -1;\n    if (this.#isStopped) {\n      return;\n    }\n    this.#isStopped = true;\n    this.#isPaused = false;\n    this.#lastLine = -1;\n    this.#backgroundPrebuildEndLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#comparisonStateStack = [];\n    this.#comparisonStateStackStart = 0;\n    this.#comparisonLineChanges = [];\n    this.#detachMessageListener();\n  }\n\n  pauseBackgroundTokenize(): void {\n    if (this.#isStopped || this.#isPaused) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization paused', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = true;\n  }\n\n  resumeBackgroundTokenize(): void {\n    if (\n      this.#isStopped ||\n      !this.#isPaused ||\n      this.#grammar == null ||\n      this.#lastLine < 0\n    ) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization resumed', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = false;\n    this.#postTokenizeMessage(this.#backgroundJobId);\n  }\n\n  #ensureGrammar(): void {\n    if (\n      this.#grammar == null &&\n      !isGrammarlessLanguage(this.#textDocument.languageId) &&\n      this.#highlighter\n        .getLoadedLanguages()\n        .includes(this.#textDocument.languageId)\n    ) {\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n  }\n\n  #attachMessageListener(): void {\n    if (this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.addEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = true;\n  }\n\n  #detachMessageListener(): void {\n    if (!this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.removeEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = false;\n  }\n\n  #postTokenizeMessage(jobId: number): void {\n    // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay\n    globalThis.postMessage({\n      type: 'tokenize',\n      tokenizerId: this.#tokenizerId,\n      jobId,\n    });\n  }\n\n  #scheduleBackgroundTokenize(\n    startLine: number,\n    changedLineRanges?: readonly [number, number][],\n    changedRangeIndex = 0\n  ): void {\n    if (isGrammarlessLanguage(this.#textDocument.languageId)) {\n      return;\n    }\n\n    const jobId = ++this.#backgroundJobId;\n\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization scheduled', {\n        jobId,\n        startLine,\n        changedLineRanges,\n        changedRangeIndex,\n      });\n    }\n\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = startLine;\n    if (this.#backgroundPrebuildEndLine >= 0) {\n      this.#pendingPrebuildEndLine = Math.max(\n        this.#pendingPrebuildEndLine,\n        this.#backgroundPrebuildEndLine\n      );\n    }\n    this.#backgroundPrebuildEndLine = -1;\n    this.#backgroundChangedLineRanges = changedLineRanges;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #scheduleStatePrebuild(endLine: number): void {\n    if (this.#grammar == null || this.#stateStack.length > endLine) {\n      return;\n    }\n    // Extend an active prebuild, or retain the target until the foreground\n    // edit job reconverges and releases the state cache.\n    if (!this.#isStopped) {\n      if (this.#backgroundPrebuildEndLine >= 0) {\n        this.#backgroundPrebuildEndLine = Math.max(\n          this.#backgroundPrebuildEndLine,\n          endLine\n        );\n      } else {\n        this.#pendingPrebuildEndLine = Math.max(\n          this.#pendingPrebuildEndLine,\n          endLine\n        );\n      }\n      return;\n    }\n\n    const jobId = ++this.#backgroundJobId;\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = this.#stateStack.length - 1;\n    this.#backgroundPrebuildEndLine = endLine;\n    this.#pendingPrebuildEndLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #tokenizeLineAt(\n    line: number,\n    state: StateStack\n  ): { resolvedTokens: Array<HighlightedToken>; state: StateStack } {\n    const lineText = this.#textDocument.getLineText(line);\n    if (lineText.length >= this.#tokenizeMaxLineLength) {\n      console.warn(\n        `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n      );\n      this.#cacheBracketIgnoredRanges(line, null);\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    if (this.#grammar == null || lineText === '' || lineText.trim() === '') {\n      this.#cacheBracketIgnoredRanges(line, null);\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    const result = tokenizeLine(\n      this.#grammar,\n      this.#colorMap,\n      lineText,\n      state,\n      TOKENIZE_TIME_LIMIT,\n      this.#matchBrackets\n    );\n    this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges);\n    return {\n      resolvedTokens: result.resolvedTokens,\n      state: result.ruleStack,\n    };\n  }\n\n  #cacheBracketIgnoredRanges(\n    line: number,\n    ranges: [number, number][] | null\n  ): void {\n    if (this.#matchBrackets) {\n      this.#bracketIgnoredRanges[line] = ranges;\n    }\n  }\n\n  // Preserve old end states as comparison-only sentinels, copying whichever\n  // side of the edit is smaller. Index shifts are resolved lazily on lookup.\n  #shiftComparisonStateStack(change: TextDocumentChange): void {\n    const lineChanges =\n      change.changedLineChanges ??\n      ([[change.startLine, change.endLine, change.lineDelta]] as const);\n    const comparisonStart = change.startLine + 1;\n    const comparisonLength = this.#stateStack.length - comparisonStart;\n    if (comparisonStart <= comparisonLength) {\n      this.#comparisonStateStack = this.#stateStack;\n      this.#comparisonStateStackStart = 0;\n      this.#stateStack = this.#stateStack.slice(0, comparisonStart);\n    } else {\n      this.#comparisonStateStackStart = comparisonStart;\n      this.#comparisonStateStack = this.#stateStack.slice(comparisonStart);\n      this.#stateStack.length = Math.min(\n        this.#stateStack.length,\n        comparisonStart\n      );\n    }\n    this.#comparisonLineChanges = lineChanges;\n  }\n\n  #getPreviousEndState(line: number): StateStack | undefined {\n    let previousLine = line;\n    for (\n      let index = this.#comparisonLineChanges.length - 1;\n      index >= 0;\n      index--\n    ) {\n      const [startLine, endLine, lineDelta] =\n        this.#comparisonLineChanges[index];\n      if (lineDelta === 0) {\n        continue;\n      }\n      if (previousLine > endLine) {\n        previousLine -= lineDelta;\n      } else if (previousLine > startLine) {\n        return this.#stateStack[line];\n      }\n    }\n    return (\n      this.#comparisonStateStack[\n        previousLine - this.#comparisonStateStackStart\n      ] ?? this.#stateStack[line]\n    );\n  }\n\n  #buildStateStack(endAt: number, timeBudget?: number): boolean {\n    const boundedEndAt = Math.min(\n      Math.max(0, endAt),\n      this.#textDocument.lineCount\n    );\n    if (this.#stateStack.length > boundedEndAt || this.#grammar == null) {\n      return true;\n    }\n    const startedAt = timeBudget == null ? 0 : performance.now();\n    let line = this.#stateStack.length - 1;\n    let state = this.#stateStack[line] ?? INITIAL;\n    while (line < boundedEndAt) {\n      this.#stateStack[line] = state;\n      const lineText = this.#textDocument.getLineText(line);\n      if (\n        lineText.length < this.#tokenizeMaxLineLength &&\n        lineText !== '' &&\n        lineText.trim() !== ''\n      ) {\n        const result = tokenizeLine(\n          this.#grammar,\n          this.#colorMap,\n          lineText,\n          state,\n          TOKENIZE_TIME_LIMIT,\n          this.#matchBrackets,\n          false\n        );\n        this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges);\n        state = result.ruleStack;\n      } else {\n        this.#cacheBracketIgnoredRanges(line, null);\n      }\n      line++;\n      this.#stateStack[line] = state;\n      if (timeBudget != null && performance.now() - startedAt > timeBudget) {\n        break;\n      }\n    }\n    return line >= boundedEndAt;\n  }\n\n  #backgroundPrebuild(jobId: number): void {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar == null ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    this.#ensureActiveTheme();\n    // State prebuilds intentionally omit rendered tokens and yield between\n    // short chunks so a deep viewport does not monopolize the main thread.\n    const complete = this.#buildStateStack(this.#backgroundPrebuildEndLine, 1);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n    if (complete) {\n      this.stopBackgroundTokenize();\n      return;\n    }\n    this.#lastLine = this.#stateStack.length - 1;\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #backgroundTokenize(jobId: number) {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar == null ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    this.#ensureActiveTheme();\n\n    const t = performance.now();\n    const lines = new Map<number, Array<HighlightedToken>>();\n    const totalLines = this.#textDocument.lineCount;\n    const changedLineRanges = this.#backgroundChangedLineRanges;\n\n    let line = this.#lastLine;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    let changedRangeIndex = this.#backgroundChangedRangeIndex;\n    let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];\n    for (; line < totalLines; ) {\n      this.#stateStack[line] = state;\n\n      const previousNextState =\n        currentChangedRangeEnd != null\n          ? this.#getPreviousEndState(line + 1)\n          : undefined;\n      const result = this.#tokenizeLineAt(line, state);\n      lines.set(line, result.resolvedTokens);\n      state = result.state;\n\n      this.#stateStack[line + 1] = state;\n      settled =\n        currentChangedRangeEnd != null &&\n        line >= currentChangedRangeEnd &&\n        previousNextState != null &&\n        state.equals(previousNextState);\n      line++;\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges?.[changedRangeIndex];\n        if (nextRange == null) {\n          break;\n        }\n        currentChangedRangeEnd = nextRange[1];\n        if (this.#stateStack[nextRange[0]] == null) {\n          settled = false;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          settled = false;\n          continue;\n        }\n      }\n\n      // limit the time of partial tokenize to 1ms\n      if (performance.now() - t > 1) {\n        break;\n      }\n    }\n\n    this.#onDeferTokenize(lines);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n\n    if (settled || line >= totalLines) {\n      const pendingPrebuildEndLine = this.#pendingPrebuildEndLine;\n      this.stopBackgroundTokenize();\n      if (pendingPrebuildEndLine >= 0) {\n        this.#scheduleStatePrebuild(pendingPrebuildEndLine);\n      }\n      return;\n    }\n\n    this.#lastLine = line;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#postTokenizeMessage(jobId);\n  }\n}\n\nfunction tokenizeLine(\n  grammar: IGrammar,\n  colorMap: string[],\n  lineText: string,\n  stateStack: StateStack,\n  timeLimit?: number,\n  collectBracketIgnoredRanges = true,\n  resolveTokens = true\n): {\n  ruleStack: StateStack;\n  resolvedTokens: Array<HighlightedToken>;\n  bracketIgnoredRanges: [number, number][];\n} {\n  const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);\n  if (result.stoppedEarly) {\n    console.warn(\n      `[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`\n    );\n  }\n  const rawTokens = result.tokens;\n  const tokensLength = rawTokens.length / 2;\n  const resolvedTokens: Array<HighlightedToken> = [];\n  const bracketIgnoredRanges: [number, number][] = [];\n  if (!resolveTokens && !collectBracketIgnoredRanges) {\n    return {\n      ruleStack: result.ruleStack,\n      resolvedTokens,\n      bracketIgnoredRanges,\n    };\n  }\n  for (let j = 0; j < tokensLength; j++) {\n    const offset = rawTokens[2 * j];\n    const nextOffset =\n      j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;\n    if (offset === nextOffset) {\n      // should never reach here, skip if happens anyway\n      continue;\n    }\n    const metadata = rawTokens[2 * j + 1];\n    if (resolveTokens) {\n      const fg = EncodedTokenMetadata.getForeground(metadata);\n      resolvedTokens.push([\n        offset,\n        colorMap[fg],\n        lineText.slice(offset, nextOffset),\n      ]);\n    }\n    if (\n      collectBracketIgnoredRanges &&\n      EncodedTokenMetadata.getTokenType(metadata) > 0\n    ) {\n      bracketIgnoredRanges.push([offset, nextOffset]);\n    }\n  }\n  return {\n    ruleStack: result.ruleStack,\n    resolvedTokens,\n    bracketIgnoredRanges,\n  };\n}\n\n// Shiki special-cases `text` and `ansi` in codeToHast but does not expose grammars.\nfunction isGrammarlessLanguage(languageId: string): boolean {\n  return languageId === 'text' || languageId === 'ansi';\n}\n"],"mappings":";;;AAgBA,MAAM,sBAAsB;AAC5B,IAAI,kBAAkB;;AAGtB,IAAa,uBAAb,MAAkE;CAwG7C;CAvGnB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,eAAe;CACf,cAA4B,CAAC,OAAO;CACpC,wBAAsC,CAAC;CACvC,6BAA6B;CAC7B,yBAEI,CAAC;CACL,YAAoB;CACpB,aAAsB;CACtB,YAAqB;CACrB,mBAA2B;CAC3B,eAAe,EAAE;CACjB,6BAA6B;CAC7B,0BAA0B;CAC1B;CACA,+BAAuC;CACvC,wBAAmE,CAAC;CACpE,6BAAsC;CAEtC,sBAAsB,SAAS,OAAO,gBAA8B;EAElE,IAAI,KAAK,cACP;EAEF,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,CAAC;EACpE,MAAM,UAAU,KAAK,IACnB,eAAe,WAAW,WAAW,eAAe,YACpD,KAAK,cAAc,SACrB;EACA,IACE,KAAK,YAAY,QACjB,CAAC,sBAAsB,KAAK,cAAc,UAAU,GACpD;GACA,MAAM,KAAK,SAAS,cAAc,CAAC,KAAK,cAAc,UAAU,CAAC;GACjE,IAAI,KAAK,cACP;GAEF,KAAK,WAAW,KAAK,aAAa,YAChC,KAAK,cAAc,UACrB;EACF;EACA,KAAK,mBAAmB;EACxB,KAAK,uBAAuB,OAAO;CACrC,GAAG,GAAG;CAEN,cAAc,EAAE,WAAkC;EAChD,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC;EAEF,MAAM,EAAE,MAAM,aAAa,UAAU;EAKrC,IACE,SAAS,cACT,gBAAgB,KAAK,gBACrB,OAAO,UAAU,YACjB,UAAU,KAAK,kBACf;GACA,IAAI,KAAK,8BAA8B,GACrC,KAAK,oBAAoB,KAAK;QAE9B,KAAK,oBAAoB,KAAK;EAElC;CACF;CAEA,mCACE,WAC2B;EAC3B,IACE,CAAC,KAAK,kBACN,YAAY,KACZ,aAAa,KAAK,cAAc,WAEhC,OAAO;EAET,KAAK,eAAe;EACpB,IAAI,KAAK,YAAY,MACnB,OAAO;EAET,IAAI,KAAK,sBAAsB,eAAe,KAAA,GAAW;GACvD,KAAK,iBAAiB,SAAS;GAC/B,MAAM,QAAQ,KAAK,YAAY,cAAc;GAC7C,MAAM,SAAS,KAAK,gBAAgB,WAAW,KAAK;GACpD,KAAK,YAAY,YAAY,KAAK,OAAO;EAC3C;EACA,OAAO,KAAK,sBAAsB,cAAc;CAClD;CAEA,YACE,aACA,SACA,SAIA;EAJiB,KAAA,UAAA;EAKjB,KAAK,eAAe;EACpB,KAAK,gBAAgB,QAAQ;EAC7B,KAAK,aAAa,QAAQ;EAC1B,KAAK,YAAY,CAAC;EAElB,KAAK,yBAAyB,QAAQ,yBAAyB;EAC/D,KAAK,mBAAmB,QAAQ;EAChC,KAAK,iBAAiB,QAAQ,kBAAkB;EAChD,KAAK,SAAS,QAAQ,WAAW;EACjC,KAAK,eAAe;EACpB,KAAK,mBAAmB;CAC1B;CAEA,SAAS,WAAyB;EAChC,IAAI,KAAK,eAAe,WAAW;EACnC,KAAK,aAAa;EAClB,KAAK,mBAAmB;EACxB,KAAK,uBAAuB;EAC5B,KAAK,cAAc,CAAC,OAAO;EAC3B,KAAK,wBAAwB,CAAC;EAC9B,KAAK,6BAA6B;EAClC,KAAK,yBAAyB,CAAC;EAC/B,IAAI,KAAK,YAAY,QAAQ,KAAK,cAAc,YAAY,GAC1D,KAAK,4BAA4B,CAAC;CAEtC;CAIA,qBAA2B;EACzB,IAAI,KAAK,eAAe,IACtB;EAGF,KAAK,SAAS,SAAS,KAAK,UAAU;EACtC,MAAM,EAAE,aAAa,KAAK,aAAa,SAAS,KAAK,UAAU;EAC/D,KAAK,YAAY;CACnB;CAEA,UAAgB;EACd,KAAK,eAAe;EACpB,KAAK,uBAAuB;EAC5B,KAAK,uBAAuB;CAC9B;CAGA,SACE,QACA,aACA,mBAAmB,OACmB;EACtC,KAAK,eAAe;EACpB,KAAK,mBAAmB;EACxB,IACE,KAAK,YAAY,QACjB,CAAC,sBAAsB,KAAK,cAAc,UAAU,GAEpD,MAAM,IAAI,MACR,yBAAyB,KAAK,cAAc,WAAW,aACzD;EAGF,MAAM,EAAE,cAAc,KAAK;EAC3B,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,CAAC;EACpE,MAAM,qBACJ,eAAe,WACX,YACA,KAAK,IAAI,eAAe,YAAY,SAAS;EAEnD,MAAM,aAAa,OAAO;EAC1B,MAAM,YAAY,KAAK,IAAI,cAAc,UAAU;EACnD,MAAM,wBACJ,eAAe,QACf,eAAe,YACf,OAAO,YAAY,KACnB,aAAa,sBACb,OAAO,WAAW;EACpB,MAAM,uBACJ,OAAO,cAAc,MACpB,OAAO,oBAAoB,OAAO,KAAK,eAAe,cAAc,CAAC,KACpE;EACJ,IAAI,KAAK,kBAAkB,CAAC,sBAE1B,KAAK,sBAAsB,SAAS,KAAK,IACvC,KAAK,sBAAsB,QAC3B,OAAO,SACT;EAEF,MAAM,wBACJ,oBAAoB,OAAO,cAAc,KAAK,cAAc;EAC9D,MAAM,0BACJ,wBAAwB,eAAe,QAAQ,cAAc;EAC/D,MAAM,oBACJ,OAAO,qBAAqB,CAAC,CAAC,YAAY,OAAO,OAAO,CAAC;EAC3D,KAAK,wBAAwB,CAAC;EAC9B,KAAK,6BAA6B;EAClC,KAAK,yBAAyB,CAAC;EAC/B,IAAI,mBAAmB;EACvB,IAAI,aAAa,WACV;QAAA,MAAM,CAAC,YAAY,aAAa,mBACnC,IAAI,aAAa,WACf,mBAAmB,KAAK,IACtB,kBACA,KAAK,IAAI,UAAU,YAAY,CAAC,CAClC;EAAA;EAIN,MAAM,4BACJ,oBAAoB,eACnB,wBAAwB,OAAO,YAAY;EAC9C,IAAI,sBACF,KAAK,iBAAiB,UAAU;OAC3B;GACL,KAAK,2BAA2B,MAAM;GACtC,IAAI,eAAe,QAAQ,cAAc,WACvC,KAAK,iBAAiB,SAAS;EAEnC;EAEA,IAAI,oBAAoB;EACxB,IAAI,yBAAyB,kBAAkB,kBAAkB,CAAC;EAClE,IAAI;EACJ,IAAI,8BAA8B;EAClC,IAAI,OAAO,uBACP,kBAAkB,kBAAkB,CAAC,KACrC;EACJ,IAAI,UAAU;EACd,MAAM,6BAAmD,IAAI,IAAI;EACjE,MAAM,sBAEU,4CAA4B,IAAI,IAAI,IAAI,KAAA;EACxD,IAAI,uBAAuB,QAAQ,CAAC,sBAAsB;GACxD,MAAM,eAAe,KAAK,IACxB,mBAAmB,GACnB,WACA,kBACF;GACA,IAAI,eAAe,YAAY;IAC7B,KAAK,iBAAiB,YAAY;IAClC,IAAI,gBAAgB;IACpB,IAAI,iBAAiB,KAAK,YAAY,kBAAkB;IACxD,OAAO,gBAAgB,cAAc,iBAAiB;KACpD,MAAM,WAAW,KAAK,gBAAgB,eAAe,cAAc;KACnE,iBAAiB,SAAS;KAC1B,oBAAoB,IAAI,eAAe,SAAS,cAAc;IAChE;IACA,KAAK,YAAY,gBAAgB;GACnC;EACF;EAGA,IAAI,QAAQ,KAAK,YAAY,SAAS;EACtC,OAAO,OAAO,qBAAsB;GAClC,MAAM,oBAAoB,uBACtB,KAAK,YAAY,OAAO,KACxB,wBACE,KAAK,qBAAqB,OAAO,CAAC,IAClC,KAAA;GACN,IAAI,yBACF,KAAK,YAAY,QAAQ;GAG3B,MAAM,EAAE,gBAAgB,OAAO,cAAc,KAAK,gBAChD,MACA,KACF;GACA,QAAQ;GAER,IAAI,QAAQ,WACV,WAAW,IAAI,MAAM,cAAc;QAEnC,qBAAqB,IAAI,MAAM,cAAc;GAG/C,IAAI,yBACF,KAAK,YAAY,OAAO,KAAK;GAE/B,UACE,QAAQ,2BACP,wBAAwB,0BACzB,qBAAqB,QACrB,MAAM,OAAO,iBAAiB;GAChC,IAAI,SAAS;IACX;IACA,MAAM,YAAY,kBAAkB;IACpC,IAAI,aAAa,MACf;IAEF,IAAI,UAAU,MAAM,oBAAoB;KACtC,sBAAsB,UAAU;KAChC,8BAA8B;KAC9B;IACF;IACA,IAAI,YAAoC,KAAK,YAAY,UAAU;IACnE,IAAI,uBACF,KACE,IAAI,YAAY,OAAO,GACvB,aAAa,UAAU,IACvB,aACA;KACA,YAAY,KAAK,qBAAqB,SAAS;KAC/C,IAAI,aAAa,MACf;KAEF,KAAK,YAAY,aAAa;IAChC;IAEF,IAAI,aAAa,MAAM;KACrB,yBAAyB,UAAU;KACnC;IACF,OAAO;KACL,OAAO,UAAU;KACjB,QAAQ;KACR,yBAAyB,UAAU;IACrC;IACA,UAAU;IACV;GACF;GACA;EACF;EAEA,IAAI,yBAAyB;GAC3B,IAAI,OAAO,oBACT,KAAK,YAAY,OAAO,KAAK;QAE7B,KAAK,YAAY,QAAQ;EAE7B;EAEA,IAAI,WAAW,yBAAyB,uBAAuB,MAAM;GACnE,KAAK,IAAI,YAAY,OAAO,GAAG,aAAa,WAAW,aAAa;IAClE,MAAM,gBAAgB,KAAK,qBAAqB,SAAS;IACzD,IAAI,iBAAiB,MACnB;IAEF,KAAK,YAAY,aAAa;GAChC;GACA,KAAK,wBAAwB,CAAC;GAC9B,KAAK,6BAA6B;GAClC,KAAK,yBAAyB,CAAC;EACjC;EAEA,IAAI,uBAAuB,QAAQ,oBAAoB,OAAO,GAC5D,KAAK,iBAAiB,mBAAmB;EAG3C,IAAI,uBAAuB,MAAM;GAC/B,IAAI,KAAK,kBAAkB,sBACzB,KAAK,sBAAsB,SAAS,KAAK,IACvC,KAAK,sBAAsB,QAC3B,mBACF;GAEF,KAAK,4BACH,qBACA,mBACA,2BACF;EACF,OAAO,IAAI,CAAC,WAAW,OAAO,WAAW;GACvC,MAAM,iBACJ,yBAAyB,cAAc,YACnC,qBACA,aAAa,aAAa,CAAC,uBACzB,aACA;GACR,IAAI,KAAK,kBAAkB,sBACzB,KAAK,sBAAsB,SAAS,KAAK,IACvC,KAAK,sBAAsB,QAC3B,cACF;GAEF,KAAK,4BACH,gBACA,mBACA,iBACF;EACF;EAEA,OAAO;CACT;CAEA,mBAAmB,aAAiC;EAClD,KAAK,eAAe;EACpB,KAAK,oBAAoB,WAAW;CACtC;CAEA,yBAA+B;EAC7B,KAAK,0BAA0B;EAC/B,IAAI,KAAK,YACP;EAEF,KAAK,aAAa;EAClB,KAAK,YAAY;EACjB,KAAK,YAAY;EACjB,KAAK,6BAA6B;EAClC,KAAK,+BAA+B,KAAA;EACpC,KAAK,+BAA+B;EACpC,KAAK,wBAAwB,CAAC;EAC9B,KAAK,6BAA6B;EAClC,KAAK,yBAAyB,CAAC;EAC/B,KAAK,uBAAuB;CAC9B;CAEA,0BAAgC;EAC9B,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,IAAI,KAAK,QACP,QAAQ,IAAI,iDAAiD,EAC3D,OAAO,KAAK,iBACd,CAAC;EAEH,KAAK,YAAY;CACnB;CAEA,2BAAiC;EAC/B,IACE,KAAK,cACL,CAAC,KAAK,aACN,KAAK,YAAY,QACjB,KAAK,YAAY,GAEjB;EAEF,IAAI,KAAK,QACP,QAAQ,IAAI,kDAAkD,EAC5D,OAAO,KAAK,iBACd,CAAC;EAEH,KAAK,YAAY;EACjB,KAAK,qBAAqB,KAAK,gBAAgB;CACjD;CAEA,iBAAuB;EACrB,IACE,KAAK,YAAY,QACjB,CAAC,sBAAsB,KAAK,cAAc,UAAU,KACpD,KAAK,aACF,mBAAmB,CAAC,CACpB,SAAS,KAAK,cAAc,UAAU,GAEzC,KAAK,WAAW,KAAK,aAAa,YAChC,KAAK,cAAc,UACrB;CAEJ;CAEA,yBAA+B;EAC7B,IAAI,KAAK,4BACP;EAEF,WAAW,iBAAiB,WAAW,KAAK,UAAU;EACtD,KAAK,6BAA6B;CACpC;CAEA,yBAA+B;EAC7B,IAAI,CAAC,KAAK,4BACR;EAEF,WAAW,oBAAoB,WAAW,KAAK,UAAU;EACzD,KAAK,6BAA6B;CACpC;CAEA,qBAAqB,OAAqB;EAExC,WAAW,YAAY;GACrB,MAAM;GACN,aAAa,KAAK;GAClB;EACF,CAAC;CACH;CAEA,4BACE,WACA,mBACA,oBAAoB,GACd;EACN,IAAI,sBAAsB,KAAK,cAAc,UAAU,GACrD;EAGF,MAAM,QAAQ,EAAE,KAAK;EAErB,IAAI,KAAK,QACP,QAAQ,IAAI,oDAAoD;GAC9D;GACA;GACA;GACA;EACF,CAAC;EAGH,KAAK,aAAa;EAClB,KAAK,YAAY;EACjB,KAAK,YAAY;EACjB,IAAI,KAAK,8BAA8B,GACrC,KAAK,0BAA0B,KAAK,IAClC,KAAK,yBACL,KAAK,0BACP;EAEF,KAAK,6BAA6B;EAClC,KAAK,+BAA+B;EACpC,KAAK,+BAA+B;EACpC,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,KAAK;CACjC;CAEA,uBAAuB,SAAuB;EAC5C,IAAI,KAAK,YAAY,QAAQ,KAAK,YAAY,SAAS,SACrD;EAIF,IAAI,CAAC,KAAK,YAAY;GACpB,IAAI,KAAK,8BAA8B,GACrC,KAAK,6BAA6B,KAAK,IACrC,KAAK,4BACL,OACF;QAEA,KAAK,0BAA0B,KAAK,IAClC,KAAK,yBACL,OACF;GAEF;EACF;EAEA,MAAM,QAAQ,EAAE,KAAK;EACrB,KAAK,aAAa;EAClB,KAAK,YAAY;EACjB,KAAK,YAAY,KAAK,YAAY,SAAS;EAC3C,KAAK,6BAA6B;EAClC,KAAK,0BAA0B;EAC/B,KAAK,+BAA+B,KAAA;EACpC,KAAK,+BAA+B;EACpC,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,KAAK;CACjC;CAEA,gBACE,MACA,OACgE;EAChE,MAAM,WAAW,KAAK,cAAc,YAAY,IAAI;EACpD,IAAI,SAAS,UAAU,KAAK,wBAAwB;GAClD,QAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,QAC1D;GACA,KAAK,2BAA2B,MAAM,IAAI;GAC1C,OAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;IAAQ,CAAC;IAAG;GAAM;EACtD;EACA,IAAI,KAAK,YAAY,QAAQ,aAAa,MAAM,SAAS,KAAK,MAAM,IAAI;GACtE,KAAK,2BAA2B,MAAM,IAAI;GAC1C,OAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;IAAQ,CAAC;IAAG;GAAM;EACtD;EACA,MAAM,SAAS,aACb,KAAK,UACL,KAAK,WACL,UACA,OACA,qBACA,KAAK,cACP;EACA,KAAK,2BAA2B,MAAM,OAAO,oBAAoB;EACjE,OAAO;GACL,gBAAgB,OAAO;GACvB,OAAO,OAAO;EAChB;CACF;CAEA,2BACE,MACA,QACM;EACN,IAAI,KAAK,gBACP,KAAK,sBAAsB,QAAQ;CAEvC;CAIA,2BAA2B,QAAkC;EAC3D,MAAM,cACJ,OAAO,sBACN,CAAC;GAAC,OAAO;GAAW,OAAO;GAAS,OAAO;EAAS,CAAC;EACxD,MAAM,kBAAkB,OAAO,YAAY;EAE3C,IAAI,mBADqB,KAAK,YAAY,SAAS,iBACV;GACvC,KAAK,wBAAwB,KAAK;GAClC,KAAK,6BAA6B;GAClC,KAAK,cAAc,KAAK,YAAY,MAAM,GAAG,eAAe;EAC9D,OAAO;GACL,KAAK,6BAA6B;GAClC,KAAK,wBAAwB,KAAK,YAAY,MAAM,eAAe;GACnE,KAAK,YAAY,SAAS,KAAK,IAC7B,KAAK,YAAY,QACjB,eACF;EACF;EACA,KAAK,yBAAyB;CAChC;CAEA,qBAAqB,MAAsC;EACzD,IAAI,eAAe;EACnB,KACE,IAAI,QAAQ,KAAK,uBAAuB,SAAS,GACjD,SAAS,GACT,SACA;GACA,MAAM,CAAC,WAAW,SAAS,aACzB,KAAK,uBAAuB;GAC9B,IAAI,cAAc,GAChB;GAEF,IAAI,eAAe,SACjB,gBAAgB;QACX,IAAI,eAAe,WACxB,OAAO,KAAK,YAAY;EAE5B;EACA,OACE,KAAK,sBACH,eAAe,KAAK,+BACjB,KAAK,YAAY;CAE1B;CAEA,iBAAiB,OAAe,YAA8B;EAC5D,MAAM,eAAe,KAAK,IACxB,KAAK,IAAI,GAAG,KAAK,GACjB,KAAK,cAAc,SACrB;EACA,IAAI,KAAK,YAAY,SAAS,gBAAgB,KAAK,YAAY,MAC7D,OAAO;EAET,MAAM,YAAY,cAAc,OAAO,IAAI,YAAY,IAAI;EAC3D,IAAI,OAAO,KAAK,YAAY,SAAS;EACrC,IAAI,QAAQ,KAAK,YAAY,SAAS;EACtC,OAAO,OAAO,cAAc;GAC1B,KAAK,YAAY,QAAQ;GACzB,MAAM,WAAW,KAAK,cAAc,YAAY,IAAI;GACpD,IACE,SAAS,SAAS,KAAK,0BACvB,aAAa,MACb,SAAS,KAAK,MAAM,IACpB;IACA,MAAM,SAAS,aACb,KAAK,UACL,KAAK,WACL,UACA,OACA,qBACA,KAAK,gBACL,KACF;IACA,KAAK,2BAA2B,MAAM,OAAO,oBAAoB;IACjE,QAAQ,OAAO;GACjB,OACE,KAAK,2BAA2B,MAAM,IAAI;GAE5C;GACA,KAAK,YAAY,QAAQ;GACzB,IAAI,cAAc,QAAQ,YAAY,IAAI,IAAI,YAAY,YACxD;EAEJ;EACA,OAAO,QAAQ;CACjB;CAEA,oBAAoB,OAAqB;EACvC,IACE,KAAK,cACL,KAAK,aACL,KAAK,YAAY,QACjB,UAAU,KAAK,kBAEf;EAGF,KAAK,mBAAmB;EAGxB,MAAM,WAAW,KAAK,iBAAiB,KAAK,4BAA4B,CAAC;EACzE,IAAI,KAAK,cAAc,KAAK,aAAa,UAAU,KAAK,kBACtD;EAEF,IAAI,UAAU;GACZ,KAAK,uBAAuB;GAC5B;EACF;EACA,KAAK,YAAY,KAAK,YAAY,SAAS;EAC3C,KAAK,qBAAqB,KAAK;CACjC;CAEA,oBAAoB,OAAe;EACjC,IACE,KAAK,cACL,KAAK,aACL,KAAK,YAAY,QACjB,UAAU,KAAK,kBAEf;EAGF,KAAK,mBAAmB;EAExB,MAAM,IAAI,YAAY,IAAI;EAC1B,MAAM,wBAAQ,IAAI,IAAqC;EACvD,MAAM,aAAa,KAAK,cAAc;EACtC,MAAM,oBAAoB,KAAK;EAE/B,IAAI,OAAO,KAAK;EAChB,IAAI,QAAQ,KAAK,YAAY,SAAS;EACtC,IAAI,UAAU;EACd,IAAI,oBAAoB,KAAK;EAC7B,IAAI,yBAAyB,oBAAoB,kBAAkB,GAAG;EACtE,OAAO,OAAO,aAAc;GAC1B,KAAK,YAAY,QAAQ;GAEzB,MAAM,oBACJ,0BAA0B,OACtB,KAAK,qBAAqB,OAAO,CAAC,IAClC,KAAA;GACN,MAAM,SAAS,KAAK,gBAAgB,MAAM,KAAK;GAC/C,MAAM,IAAI,MAAM,OAAO,cAAc;GACrC,QAAQ,OAAO;GAEf,KAAK,YAAY,OAAO,KAAK;GAC7B,UACE,0BAA0B,QAC1B,QAAQ,0BACR,qBAAqB,QACrB,MAAM,OAAO,iBAAiB;GAChC;GACA,IAAI,SAAS;IACX;IACA,MAAM,YAAY,oBAAoB;IACtC,IAAI,aAAa,MACf;IAEF,yBAAyB,UAAU;IACnC,IAAI,KAAK,YAAY,UAAU,OAAO,MACpC,UAAU;SACL;KACL,OAAO,UAAU;KACjB,QAAQ,KAAK,YAAY,SAAS;KAClC,UAAU;KACV;IACF;GACF;GAGA,IAAI,YAAY,IAAI,IAAI,IAAI,GAC1B;EAEJ;EAEA,KAAK,iBAAiB,KAAK;EAC3B,IAAI,KAAK,cAAc,KAAK,aAAa,UAAU,KAAK,kBACtD;EAGF,IAAI,WAAW,QAAQ,YAAY;GACjC,MAAM,yBAAyB,KAAK;GACpC,KAAK,uBAAuB;GAC5B,IAAI,0BAA0B,GAC5B,KAAK,uBAAuB,sBAAsB;GAEpD;EACF;EAEA,KAAK,YAAY;EACjB,KAAK,+BAA+B;EACpC,KAAK,qBAAqB,KAAK;CACjC;AACF;AAEA,SAAS,aACP,SACA,UACA,UACA,YACA,WACA,8BAA8B,MAC9B,gBAAgB,MAKhB;CACA,MAAM,SAAS,QAAQ,cAAc,UAAU,YAAY,SAAS;CACpE,IAAI,OAAO,cACT,QAAQ,KACN,oDAAoD,SAAS,UAAU,GAAG,GAAG,GAC/E;CAEF,MAAM,YAAY,OAAO;CACzB,MAAM,eAAe,UAAU,SAAS;CACxC,MAAM,iBAA0C,CAAC;CACjD,MAAM,uBAA2C,CAAC;CAClD,IAAI,CAAC,iBAAiB,CAAC,6BACrB,OAAO;EACL,WAAW,OAAO;EAClB;EACA;CACF;CAEF,KAAK,IAAI,IAAI,GAAG,IAAI,cAAc,KAAK;EACrC,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,aACJ,IAAI,IAAI,eAAe,UAAU,IAAI,IAAI,KAAK,SAAS;EACzD,IAAI,WAAW,YAEb;EAEF,MAAM,WAAW,UAAU,IAAI,IAAI;EACnC,IAAI,eAAe;GACjB,MAAM,KAAK,qBAAqB,cAAc,QAAQ;GACtD,eAAe,KAAK;IAClB;IACA,SAAS;IACT,SAAS,MAAM,QAAQ,UAAU;GACnC,CAAC;EACH;EACA,IACE,+BACA,qBAAqB,aAAa,QAAQ,IAAI,GAE9C,qBAAqB,KAAK,CAAC,QAAQ,UAAU,CAAC;CAElD;CACA,OAAO;EACL,WAAW,OAAO;EAClB;EACA;CACF;AACF;AAGA,SAAS,sBAAsB,YAA6B;CAC1D,OAAO,eAAe,UAAU,eAAe;AACjD"}

package/dist/highlighter/shiki-stream.d.ts

@@ -0,0 +1,20 @@
+import { ThemedToken } from "./types.js";
+import { BaseStreamTokenizer } from "./stream-tokenizer.js";
+import { CodeToTokensOptions, HighlighterCore } from "shiki/core";
+//#region src/highlighter/shiki-stream.d.ts
+declare class ShikiStreamTokenizer extends BaseStreamTokenizer {
+  #private;
+  private readonly highlighter;
+  private readonly options;
+  constructor(highlighter: HighlighterCore, options: CodeToTokensOptions);
+  protected tokenizeChunk(chunk: string, stable: ThemedToken[]): {
+    unstable: ThemedToken[];
+    tailLength: number;
+  };
+  clone(): ShikiStreamTokenizer;
+  protected resetSource(): void;
+  protected releaseSource(): void;
+}
+//#endregion
+export { ShikiStreamTokenizer };
+//# sourceMappingURL=shiki-stream.d.ts.map

package/dist/highlighter/shiki-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shiki-stream.d.ts","names":[],"sources":["../../src/highlighter/shiki-stream.ts"],"mappings":";;;;cAUa,6BAA6B;;mBAMrB;mBACA;EAFnB,YACmB,aAAa,iBACb,SAAS;YAKlB,cACR,eACA,QAAQ;IACL,UAAU;IAAe;;EAyB9B,SAAS;YASC;YAKA"}

package/dist/highlighter/shiki-stream.js

@@ -0,0 +1,60 @@
+import { appendItems } from "../utils/appendItems.js";
+import { BaseStreamTokenizer } from "./stream-tokenizer.js";
+//#region src/highlighter/shiki-stream.ts
+var ShikiStreamTokenizer = class ShikiStreamTokenizer extends BaseStreamTokenizer {
+	highlighter;
+	options;
+	#tail = "";
+	#grammarState;
+	constructor(highlighter, options) {
+		super();
+		this.highlighter = highlighter;
+		this.options = options;
+	}
+	tokenizeChunk(chunk, stable) {
+		const chunkLines = (this.#tail + chunk).split(/(\r\n|\r|\n)/);
+		let unstable = [];
+		for (let i = 0; i < chunkLines.length; i += 2) {
+			const line = chunkLines[i];
+			const result = this.highlighter.codeToTokens(line, {
+				...this.options,
+				grammarState: this.#grammarState
+			});
+			const tokens = result.tokens[0].map((token) => ({
+				...token,
+				offset: token.offset + this.stableOffset
+			}));
+			if (i + 1 < chunkLines.length) {
+				appendItems(stable, tokens);
+				this.pushLineBreak(stable, line.length, chunkLines[i + 1]);
+				this.#grammarState = result.grammarState;
+			} else {
+				unstable = tokens;
+				this.#tail = line;
+			}
+		}
+		return {
+			unstable,
+			tailLength: this.#tail.length
+		};
+	}
+	clone() {
+		this.assertActive();
+		const clone = new ShikiStreamTokenizer(this.highlighter, this.options);
+		this.copyStateTo(clone);
+		clone.#tail = this.#tail;
+		clone.#grammarState = this.#grammarState;
+		return clone;
+	}
+	resetSource() {
+		this.#tail = "";
+		this.#grammarState = void 0;
+	}
+	releaseSource() {
+		this.resetSource();
+	}
+};
+//#endregion
+export { ShikiStreamTokenizer };
+
+//# sourceMappingURL=shiki-stream.js.map

package/dist/highlighter/shiki-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shiki-stream.js","names":[],"sources":["../../src/highlighter/shiki-stream.ts"],"sourcesContent":["import type {\n  CodeToTokensOptions,\n  GrammarState,\n  HighlighterCore,\n} from 'shiki/core';\n\nimport { appendItems } from '../utils/appendItems';\nimport { BaseStreamTokenizer } from './stream-tokenizer';\nimport type { ThemedToken } from './types';\n\nexport class ShikiStreamTokenizer extends BaseStreamTokenizer {\n  // Text of the provisional line; it is tokenized again with each chunk.\n  #tail = '';\n  #grammarState: GrammarState | undefined;\n\n  constructor(\n    private readonly highlighter: HighlighterCore,\n    private readonly options: CodeToTokensOptions\n  ) {\n    super();\n  }\n\n  protected tokenizeChunk(\n    chunk: string,\n    stable: ThemedToken[]\n  ): { unstable: ThemedToken[]; tailLength: number } {\n    const chunkLines = (this.#tail + chunk).split(/(\\r\\n|\\r|\\n)/);\n    let unstable: ThemedToken[] = [];\n    for (let i = 0; i < chunkLines.length; i += 2) {\n      const line = chunkLines[i];\n      const result = this.highlighter.codeToTokens(line, {\n        ...this.options,\n        grammarState: this.#grammarState,\n      });\n      const tokens = result.tokens[0].map((token) => ({\n        ...token,\n        offset: token.offset + this.stableOffset,\n      }));\n      if (i + 1 < chunkLines.length) {\n        appendItems(stable, tokens);\n        this.pushLineBreak(stable, line.length, chunkLines[i + 1]);\n        this.#grammarState = result.grammarState;\n      } else {\n        unstable = tokens;\n        this.#tail = line;\n      }\n    }\n    return { unstable, tailLength: this.#tail.length };\n  }\n\n  clone(): ShikiStreamTokenizer {\n    this.assertActive();\n    const clone = new ShikiStreamTokenizer(this.highlighter, this.options);\n    this.copyStateTo(clone);\n    clone.#tail = this.#tail;\n    clone.#grammarState = this.#grammarState;\n    return clone;\n  }\n\n  protected resetSource(): void {\n    this.#tail = '';\n    this.#grammarState = undefined;\n  }\n\n  protected releaseSource(): void {\n    this.resetSource();\n  }\n}\n"],"mappings":";;;AAUA,IAAa,uBAAb,MAAa,6BAA6B,oBAAoB;CAMzC;CACA;CALnB,QAAQ;CACR;CAEA,YACE,aACA,SACA;EACA,MAAM;EAHW,KAAA,cAAA;EACA,KAAA,UAAA;CAGnB;CAEA,cACE,OACA,QACiD;EACjD,MAAM,cAAc,KAAK,QAAQ,MAAA,CAAO,MAAM,cAAc;EAC5D,IAAI,WAA0B,CAAC;EAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;GAC7C,MAAM,OAAO,WAAW;GACxB,MAAM,SAAS,KAAK,YAAY,aAAa,MAAM;IACjD,GAAG,KAAK;IACR,cAAc,KAAK;GACrB,CAAC;GACD,MAAM,SAAS,OAAO,OAAO,EAAE,CAAC,KAAK,WAAW;IAC9C,GAAG;IACH,QAAQ,MAAM,SAAS,KAAK;GAC9B,EAAE;GACF,IAAI,IAAI,IAAI,WAAW,QAAQ;IAC7B,YAAY,QAAQ,MAAM;IAC1B,KAAK,cAAc,QAAQ,KAAK,QAAQ,WAAW,IAAI,EAAE;IACzD,KAAK,gBAAgB,OAAO;GAC9B,OAAO;IACL,WAAW;IACX,KAAK,QAAQ;GACf;EACF;EACA,OAAO;GAAE;GAAU,YAAY,KAAK,MAAM;EAAO;CACnD;CAEA,QAA8B;EAC5B,KAAK,aAAa;EAClB,MAAM,QAAQ,IAAI,qBAAqB,KAAK,aAAa,KAAK,OAAO;EACrE,KAAK,YAAY,KAAK;EACtB,MAAM,QAAQ,KAAK;EACnB,MAAM,gBAAgB,KAAK;EAC3B,OAAO;CACT;CAEA,cAA8B;EAC5B,KAAK,QAAQ;EACb,KAAK,gBAAgB,KAAA;CACvB;CAEA,gBAAgC;EAC9B,KAAK,YAAY;CACnB;AACF"}