@pierre/diffs 1.3.0-beta.5 → 1.3.0-beta.7

package/dist/editor/textMeasure.js

@@ -1,16 +1,18 @@
-import { h, round } from "./utils.js";
-
+import { getGraphemeSegmenter, h, round } from "./utils.js";
 //#region src/editor/textMeasure.ts
+const TEXT_WIDTH_CACHE_LIMIT = 4096;
 var Metrics = class {
 	#root;
 	#canvasCtx;
 	#font;
+	#textWidthCache = /* @__PURE__ */ new Map();
 	/** Width of the '0' character. */
 	ch = -1;
 	/** Size of a tab(\t) character. */
 	tabSize = 2;
 	/** Height of the code line. */
 	lineHeight = 20;
+	/** Padding top of the root element. */
 	paddingTop = 0;
 	/** initialize the metrics */
 	init(root) {
@@ -31,12 +33,34 @@ var Metrics = class {
 			this.#font = font;
 			this.#canvasCtx.font = font;
 			this.ch = this.canvasMeasureTextWidth("0");
+			this.clearTextWidthCache();
 		}
-		this.tabSize = parseInt(tabSize, 10);
+		const nextTabSize = parseInt(tabSize, 10);
+		if (!Number.isNaN(nextTabSize)) this.tabSize = nextTabSize;
+	}
+	/**
+	* Re-measure the '0' character width against the font that is loaded right
+	* now, returning true when it changed.
+	*
+	* A custom web font can finish loading after the editor first renders.
+	* Until then canvas measureText reports the fallback font's width, and
+	* getComputedStyle returns the same font-family string before and after the
+	* file arrives, so init()'s font guard never re-measures on its own. Call
+	* this once fonts have settled (e.g. on document.fonts.ready) to replace a
+	* width measured against the fallback font with the real glyph width. The
+	* boolean return lets the caller skip re-rendering when nothing changed.
+	*/
+	remeasureCharacterWidth() {
+		if (this.#canvasCtx === void 0 || this.#font === void 0) return false;
+		this.#canvasCtx.font = this.#font;
+		const ch = this.canvasMeasureTextWidth("0");
+		if (ch === this.ch) return false;
+		this.ch = ch;
+		return true;
 	}
 	/** measure the width of the text */
 	measureTextWidth(text) {
-		const textWithExpandedTabs = text.replaceAll("	", " ".repeat(this.tabSize));
+		const textWithExpandedTabs = expandTabsToSpaces(text, this.tabSize);
 		if (needsDomTextMeasurement(textWithExpandedTabs)) return this.domMeasureTextWidth(textWithExpandedTabs);
 		return this.canvasMeasureTextWidth(textWithExpandedTabs);
 	}
@@ -47,10 +71,14 @@ var Metrics = class {
 	}
 	/**
 	* measure the width of the text using the DOM
-	* this is slow because it cause a reflow, use it for non-ascii text
+	* this is slow because it cause a reflow, use it for non-ascii text;
+	* results are memoized per text so repeated measurements skip the reflow
 	*/
 	domMeasureTextWidth(text) {
 		if (this.#root === void 0) throw new Error("Metrics not initialized");
+		const cacheKey = text + "|" + this.#font;
+		const cached = this.#textWidthCache.get(cacheKey);
+		if (cached !== void 0) return cached;
 		const measureEl = h("span", {
 			style: {
 				position: "absolute",
@@ -63,11 +91,26 @@ var Metrics = class {
 			},
 			textContent: text
 		}, this.#root);
+		let width;
 		try {
-			return measureEl.getBoundingClientRect().width;
+			width = round(measureEl.getBoundingClientRect().width);
 		} finally {
 			measureEl.remove();
 		}
+		if (this.#textWidthCache.size >= TEXT_WIDTH_CACHE_LIMIT) {
+			const oldestKey = this.#textWidthCache.keys().next().value;
+			if (oldestKey !== void 0) this.#textWidthCache.delete(oldestKey);
+		}
+		this.#textWidthCache.set(cacheKey, width);
+		return width;
+	}
+	/**
+	* discard memoized DOM text widths
+	* call this when the inherited font may have changed without re-running
+	* init(), e.g. on a layout reflow, so stale widths are not reused
+	*/
+	clearTextWidthCache() {
+		this.#textWidthCache.clear();
 	}
 };
 /** Check if the text needs DOM text measurement. */
@@ -82,12 +125,22 @@ function needsDomTextMeasurement(text) {
 function snapTextOffsetToUnicodeBoundary(text, offset) {
 	const boundedOffset = Math.max(0, Math.min(offset, text.length));
 	if (boundedOffset === 0 || boundedOffset === text.length || !needsDomTextMeasurement(text)) return boundedOffset;
-	const segmenter = new Intl.Segmenter(void 0, { granularity: "grapheme" });
-	for (const segment of segmenter.segment(text)) {
-		const segmentStart = segment.index;
-		const segmentEnd = segmentStart + segment.segment.length;
+	const segmenter = getGraphemeSegmenter();
+	if (segmenter !== void 0) {
+		for (const segment of segmenter.segment(text)) {
+			const segmentStart = segment.index;
+			const segmentEnd = segmentStart + segment.segment.length;
+			if (boundedOffset > segmentStart && boundedOffset < segmentEnd) return segmentEnd;
+			if (boundedOffset <= segmentStart) break;
+		}
+		return boundedOffset;
+	}
+	let segmentStart = 0;
+	for (const codePoint of text) {
+		const segmentEnd = segmentStart + codePoint.length;
 		if (boundedOffset > segmentStart && boundedOffset < segmentEnd) return segmentEnd;
 		if (boundedOffset <= segmentStart) break;
+		segmentStart = segmentEnd;
 	}
 	return boundedOffset;
 }
@@ -95,20 +148,54 @@ function snapTextOffsetToUnicodeBoundary(text, offset) {
 function getUnicodeMeasurementOffsets(text) {
 	if (!needsDomTextMeasurement(text)) return;
 	const offsets = [0];
-	const segmenter = new Intl.Segmenter(void 0, { granularity: "grapheme" });
-	for (const segment of segmenter.segment(text)) offsets.push(segment.index + segment.segment.length);
+	const segmenter = getGraphemeSegmenter();
+	if (segmenter !== void 0) {
+		for (const segment of segmenter.segment(text)) offsets.push(segment.index + segment.segment.length);
+		return offsets;
+	}
+	let offset = 0;
+	for (const codePoint of text) {
+		offset += codePoint.length;
+		offsets.push(offset);
+	}
 	return offsets;
 }
-/** get the number of columns of the ASCII text */
+/**
+* Expand tab characters to spaces using fixed tab stops: each tab advances to
+* the next multiple of tabSize from its running column, matching how the
+* rendered text expands tabs via CSS `tab-size`. Expanding every tab to a flat
+* tabSize would mis-measure tabs that follow other characters on the same line
+* (e.g. an alignment tab in `foo\tbar`).
+*/
+function expandTabsToSpaces(text, tabSize) {
+	if (!text.includes("	")) return text;
+	let result = "";
+	let column = 0;
+	for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) === 9) {
+		const advance = tabSize - column % tabSize;
+		result += " ".repeat(advance);
+		column += advance;
+	} else {
+		result += text[i];
+		column += 1;
+	}
+	return result;
+}
+/**
+* Count the rendered columns of ASCII text, advancing each tab to the next
+* fixed tab stop (a multiple of tabSize) to match CSS `tab-size`. Returns -1
+* for non-ASCII text, which must be measured glyph-by-glyph instead.
+*/
 function getExpandedAsciiTextColumns(text, tabSize) {
 	let columns = 0;
 	for (let i = 0; i < text.length; i++) {
-		if (text.charCodeAt(i) > 127) return -1;
-		columns += text.charCodeAt(i) === 9 ? tabSize : 1;
+		const code = text.charCodeAt(i);
+		if (code > 127) return -1;
+		columns += code === 9 ? tabSize - columns % tabSize : 1;
 	}
 	return columns;
 }
-
 //#endregion
-export { Metrics, getExpandedAsciiTextColumns, getUnicodeMeasurementOffsets, needsDomTextMeasurement, snapTextOffsetToUnicodeBoundary };
+export { Metrics, expandTabsToSpaces, getExpandedAsciiTextColumns, getUnicodeMeasurementOffsets, needsDomTextMeasurement, snapTextOffsetToUnicodeBoundary };
+
 //# sourceMappingURL=textMeasure.js.map

package/dist/editor/textMeasure.js.map

@@ -1 +1 @@
-{"version":3,"file":"textMeasure.js","names":["#root","#canvasCtx","#font"],"sources":["../../src/editor/textMeasure.ts"],"sourcesContent":["import { h, round } from './utils';\n\nexport class Metrics {\n  #root?: HTMLElement;\n  #canvasCtx?: CanvasRenderingContext2D;\n  #font?: string;\n\n  /** Width of the '0' character. */\n  ch: number = -1;\n  /** Size of a tab(\\t) character. */\n  tabSize: number = 2;\n  /** Height of the code line. */\n  lineHeight: number = 20;\n\n  paddingTop: number = 0;\n\n  /** initialize the metrics */\n  init(root: HTMLElement): void {\n    if (\n      this.#root === root &&\n      this.#canvasCtx !== undefined &&\n      this.ch !== -1\n    ) {\n      // already initialized\n      return;\n    }\n\n    this.#root = root;\n    this.#canvasCtx ??=\n      document.createElement('canvas').getContext('2d') ?? undefined;\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Could not get canvas context');\n    }\n\n    const parent = root.parentElement;\n    if (parent !== null) {\n      const { paddingTop } = getComputedStyle(parent);\n      if (paddingTop.endsWith('px')) {\n        this.paddingTop = parseFloat(paddingTop.slice(0, -2));\n      }\n    }\n\n    const { fontSize, fontFamily, tabSize, lineHeight } =\n      getComputedStyle(root);\n    if (lineHeight.endsWith('px')) {\n      this.lineHeight = parseFloat(lineHeight.slice(0, -2));\n    } else if (fontSize.endsWith('px')) {\n      this.lineHeight = round(\n        parseFloat(fontSize.slice(0, -2)) * parseFloat(lineHeight)\n      );\n    }\n    const font = fontSize + ' ' + fontFamily;\n    if (this.#font !== font || this.ch === -1) {\n      this.#font = font;\n      this.#canvasCtx.font = font;\n      this.ch = this.canvasMeasureTextWidth('0');\n    }\n    this.tabSize = parseInt(tabSize, 10);\n  }\n\n  /** measure the width of the text */\n  measureTextWidth(text: string): number {\n    const textWithExpandedTabs = text.replaceAll(\n      '\\t',\n      ' '.repeat(this.tabSize)\n    );\n    if (needsDomTextMeasurement(textWithExpandedTabs)) {\n      return this.domMeasureTextWidth(textWithExpandedTabs);\n    }\n    return this.canvasMeasureTextWidth(textWithExpandedTabs);\n  }\n\n  /** measure the width of the text using the canvas measureText API */\n  canvasMeasureTextWidth(text: string): number {\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    return round(this.#canvasCtx.measureText(text).width);\n  }\n\n  /**\n   * measure the width of the text using the DOM\n   * this is slow because it cause a reflow, use it for non-ascii text\n   */\n  domMeasureTextWidth(text: string): number {\n    if (this.#root === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    const measureEl = h(\n      'span',\n      {\n        style: {\n          position: 'absolute',\n          top: '0',\n          left: '0',\n          visibility: 'hidden',\n          pointerEvents: 'none',\n          whiteSpace: 'pre',\n          font: 'inherit',\n        },\n        textContent: text,\n      },\n      this.#root\n    );\n    try {\n      return measureEl.getBoundingClientRect().width;\n    } finally {\n      measureEl.remove();\n    }\n  }\n}\n\n/** Check if the text needs DOM text measurement. */\nexport function needsDomTextMeasurement(text: string): boolean {\n  for (let i = 0; i < text.length; i++) {\n    const code = text.charCodeAt(i);\n    if (\n      (code >= 0xd800 && code <= 0xdfff) ||\n      code === 0x200d ||\n      code === 0xfe0e ||\n      code === 0xfe0f\n    ) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/** snap the text offset to the Unicode boundary */\nexport function snapTextOffsetToUnicodeBoundary(\n  text: string,\n  offset: number\n): number {\n  const boundedOffset = Math.max(0, Math.min(offset, text.length));\n  if (\n    boundedOffset === 0 ||\n    boundedOffset === text.length ||\n    !needsDomTextMeasurement(text)\n  ) {\n    return boundedOffset;\n  }\n  // Avoid measuring a caret position inside one visual emoji/grapheme.\n  // Browser caret movement can report offsets around UTF-16 surrogate\n  // pairs and emoji joiners; measuring a partial sequence gives a\n  // replacement-glyph width.\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    const segmentStart = segment.index;\n    const segmentEnd = segmentStart + segment.segment.length;\n    if (boundedOffset > segmentStart && boundedOffset < segmentEnd) {\n      return segmentEnd;\n    }\n    if (boundedOffset <= segmentStart) {\n      break;\n    }\n  }\n  return boundedOffset;\n}\n\n/** get the offsets of the Unicode grapheme clusters in the text */\nexport function getUnicodeMeasurementOffsets(\n  text: string\n): number[] | undefined {\n  if (!needsDomTextMeasurement(text)) {\n    return undefined;\n  }\n  const offsets = [0];\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    offsets.push(segment.index + segment.segment.length);\n  }\n  return offsets;\n}\n\n/** get the number of columns of the ASCII text */\nexport function getExpandedAsciiTextColumns(\n  text: string,\n  tabSize: number\n): number {\n  let columns = 0;\n  for (let i = 0; i < text.length; i++) {\n    if (text.charCodeAt(i) > 127) {\n      return -1;\n    }\n    columns += text.charCodeAt(i) === /* '\\t' */ 9 ? tabSize : 1;\n  }\n  return columns;\n}\n"],"mappings":";;;AAEA,IAAa,UAAb,MAAqB;CACnB;CACA;CACA;;CAGA,KAAa;;CAEb,UAAkB;;CAElB,aAAqB;CAErB,aAAqB;;CAGrB,KAAK,MAAyB;AAC5B,MACE,MAAKA,SAAU,QACf,MAAKC,cAAe,UACpB,KAAK,OAAO,GAGZ;AAGF,QAAKD,OAAQ;AACb,QAAKC,cACH,SAAS,cAAc,SAAS,CAAC,WAAW,KAAK,IAAI;AACvD,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,+BAA+B;EAGjD,MAAM,SAAS,KAAK;AACpB,MAAI,WAAW,MAAM;GACnB,MAAM,EAAE,eAAe,iBAAiB,OAAO;AAC/C,OAAI,WAAW,SAAS,KAAK,CAC3B,MAAK,aAAa,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;;EAIzD,MAAM,EAAE,UAAU,YAAY,SAAS,eACrC,iBAAiB,KAAK;AACxB,MAAI,WAAW,SAAS,KAAK,CAC3B,MAAK,aAAa,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;WAC5C,SAAS,SAAS,KAAK,CAChC,MAAK,aAAa,MAChB,WAAW,SAAS,MAAM,GAAG,GAAG,CAAC,GAAG,WAAW,WAAW,CAC3D;EAEH,MAAM,OAAO,WAAW,MAAM;AAC9B,MAAI,MAAKC,SAAU,QAAQ,KAAK,OAAO,IAAI;AACzC,SAAKA,OAAQ;AACb,SAAKD,UAAW,OAAO;AACvB,QAAK,KAAK,KAAK,uBAAuB,IAAI;;AAE5C,OAAK,UAAU,SAAS,SAAS,GAAG;;;CAItC,iBAAiB,MAAsB;EACrC,MAAM,uBAAuB,KAAK,WAChC,KACA,IAAI,OAAO,KAAK,QAAQ,CACzB;AACD,MAAI,wBAAwB,qBAAqB,CAC/C,QAAO,KAAK,oBAAoB,qBAAqB;AAEvD,SAAO,KAAK,uBAAuB,qBAAqB;;;CAI1D,uBAAuB,MAAsB;AAC3C,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,0BAA0B;AAE5C,SAAO,MAAM,MAAKA,UAAW,YAAY,KAAK,CAAC,MAAM;;;;;;CAOvD,oBAAoB,MAAsB;AACxC,MAAI,MAAKD,SAAU,OACjB,OAAM,IAAI,MAAM,0BAA0B;EAE5C,MAAM,YAAY,EAChB,QACA;GACE,OAAO;IACL,UAAU;IACV,KAAK;IACL,MAAM;IACN,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,MAAM;IACP;GACD,aAAa;GACd,EACD,MAAKA,KACN;AACD,MAAI;AACF,UAAO,UAAU,uBAAuB,CAAC;YACjC;AACR,aAAU,QAAQ;;;;;AAMxB,SAAgB,wBAAwB,MAAuB;AAC7D,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,OAAO,KAAK,WAAW,EAAE;AAC/B,MACG,QAAQ,SAAU,QAAQ,SAC3B,SAAS,QACT,SAAS,SACT,SAAS,MAET,QAAO;;AAGX,QAAO;;;AAIT,SAAgB,gCACd,MACA,QACQ;CACR,MAAM,gBAAgB,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,OAAO,CAAC;AAChE,KACE,kBAAkB,KAClB,kBAAkB,KAAK,UACvB,CAAC,wBAAwB,KAAK,CAE9B,QAAO;CAMT,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,EAAE;EAC7C,MAAM,eAAe,QAAQ;EAC7B,MAAM,aAAa,eAAe,QAAQ,QAAQ;AAClD,MAAI,gBAAgB,gBAAgB,gBAAgB,WAClD,QAAO;AAET,MAAI,iBAAiB,aACnB;;AAGJ,QAAO;;;AAIT,SAAgB,6BACd,MACsB;AACtB,KAAI,CAAC,wBAAwB,KAAK,CAChC;CAEF,MAAM,UAAU,CAAC,EAAE;CACnB,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,CAC3C,SAAQ,KAAK,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAEtD,QAAO;;;AAIT,SAAgB,4BACd,MACA,SACQ;CACR,IAAI,UAAU;AACd,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,MAAI,KAAK,WAAW,EAAE,GAAG,IACvB,QAAO;AAET,aAAW,KAAK,WAAW,EAAE,KAAgB,IAAI,UAAU;;AAE7D,QAAO"}
+{"version":3,"file":"textMeasure.js","names":["#root","#canvasCtx","#font","#textWidthCache"],"sources":["../../src/editor/textMeasure.ts"],"sourcesContent":["import { getGraphemeSegmenter, h, round } from './utils';\n\n// Upper bound on cached DOM text-width measurements. The cache only holds\n// non-ASCII runs (emoji, ZWJ sequences, variation selectors), so it stays\n// small for ordinary code, but capping it prevents unbounded growth on\n// emoji-heavy documents. Past the cap the oldest entry is evicted.\nconst TEXT_WIDTH_CACHE_LIMIT = 4096;\n\nexport class Metrics {\n  #root?: HTMLElement;\n  #canvasCtx?: CanvasRenderingContext2D;\n  #font?: string;\n\n  // Memoizes domMeasureTextWidth() results\n  #textWidthCache = new Map<string, number>();\n\n  /** Width of the '0' character. */\n  ch: number = -1;\n  /** Size of a tab(\\t) character. */\n  tabSize: number = 2;\n  /** Height of the code line. */\n  lineHeight: number = 20;\n  /** Padding top of the root element. */\n  paddingTop: number = 0;\n\n  /** initialize the metrics */\n  init(root: HTMLElement): void {\n    if (\n      this.#root === root &&\n      this.#canvasCtx !== undefined &&\n      this.ch !== -1\n    ) {\n      // already initialized\n      return;\n    }\n\n    this.#root = root;\n    this.#canvasCtx ??=\n      document.createElement('canvas').getContext('2d') ?? undefined;\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Could not get canvas context');\n    }\n\n    const parent = root.parentElement;\n    if (parent !== null) {\n      const { paddingTop } = getComputedStyle(parent);\n      if (paddingTop.endsWith('px')) {\n        this.paddingTop = parseFloat(paddingTop.slice(0, -2));\n      }\n    }\n\n    const { fontSize, fontFamily, tabSize, lineHeight } =\n      getComputedStyle(root);\n    if (lineHeight.endsWith('px')) {\n      this.lineHeight = parseFloat(lineHeight.slice(0, -2));\n    } else if (fontSize.endsWith('px')) {\n      this.lineHeight = round(\n        parseFloat(fontSize.slice(0, -2)) * parseFloat(lineHeight)\n      );\n    }\n    const font = fontSize + ' ' + fontFamily;\n    if (this.#font !== font || this.ch === -1) {\n      this.#font = font;\n      this.#canvasCtx.font = font;\n      this.ch = this.canvasMeasureTextWidth('0');\n      // Cached DOM widths were measured against the previous font.\n      this.clearTextWidthCache();\n    }\n    const nextTabSize = parseInt(tabSize, 10);\n    if (!Number.isNaN(nextTabSize)) {\n      this.tabSize = nextTabSize;\n    }\n  }\n\n  /**\n   * Re-measure the '0' character width against the font that is loaded right\n   * now, returning true when it changed.\n   *\n   * A custom web font can finish loading after the editor first renders.\n   * Until then canvas measureText reports the fallback font's width, and\n   * getComputedStyle returns the same font-family string before and after the\n   * file arrives, so init()'s font guard never re-measures on its own. Call\n   * this once fonts have settled (e.g. on document.fonts.ready) to replace a\n   * width measured against the fallback font with the real glyph width. The\n   * boolean return lets the caller skip re-rendering when nothing changed.\n   */\n  remeasureCharacterWidth(): boolean {\n    if (this.#canvasCtx === undefined || this.#font === undefined) {\n      return false;\n    }\n    this.#canvasCtx.font = this.#font;\n    const ch = this.canvasMeasureTextWidth('0');\n    if (ch === this.ch) {\n      return false;\n    }\n    this.ch = ch;\n    return true;\n  }\n\n  /** measure the width of the text */\n  measureTextWidth(text: string): number {\n    const textWithExpandedTabs = expandTabsToSpaces(text, this.tabSize);\n    if (needsDomTextMeasurement(textWithExpandedTabs)) {\n      return this.domMeasureTextWidth(textWithExpandedTabs);\n    }\n    return this.canvasMeasureTextWidth(textWithExpandedTabs);\n  }\n\n  /** measure the width of the text using the canvas measureText API */\n  canvasMeasureTextWidth(text: string): number {\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    return round(this.#canvasCtx.measureText(text).width);\n  }\n\n  /**\n   * measure the width of the text using the DOM\n   * this is slow because it cause a reflow, use it for non-ascii text;\n   * results are memoized per text so repeated measurements skip the reflow\n   */\n  domMeasureTextWidth(text: string): number {\n    if (this.#root === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    const cacheKey = text + '|' + this.#font;\n    const cached = this.#textWidthCache.get(cacheKey);\n    if (cached !== undefined) {\n      return cached;\n    }\n    const measureEl = h(\n      'span',\n      {\n        style: {\n          position: 'absolute',\n          top: '0',\n          left: '0',\n          visibility: 'hidden',\n          pointerEvents: 'none',\n          whiteSpace: 'pre',\n          font: 'inherit',\n        },\n        textContent: text,\n      },\n      this.#root\n    );\n    let width: number;\n    try {\n      // round() to match canvasMeasureTextWidth and ch; otherwise the DOM path\n      // returns raw sub-pixel widths and caret/selection offsets drift between\n      // ASCII and non-ASCII runs on the same line.\n      width = round(measureEl.getBoundingClientRect().width);\n    } finally {\n      measureEl.remove();\n    }\n    if (this.#textWidthCache.size >= TEXT_WIDTH_CACHE_LIMIT) {\n      const oldestKey = this.#textWidthCache.keys().next().value;\n      if (oldestKey !== undefined) {\n        this.#textWidthCache.delete(oldestKey);\n      }\n    }\n    this.#textWidthCache.set(cacheKey, width);\n    return width;\n  }\n\n  /**\n   * discard memoized DOM text widths\n   * call this when the inherited font may have changed without re-running\n   * init(), e.g. on a layout reflow, so stale widths are not reused\n   */\n  clearTextWidthCache(): void {\n    this.#textWidthCache.clear();\n  }\n}\n\n/** Check if the text needs DOM text measurement. */\nexport function needsDomTextMeasurement(text: string): boolean {\n  for (let i = 0; i < text.length; i++) {\n    const code = text.charCodeAt(i);\n    if (\n      (code >= 0xd800 && code <= 0xdfff) ||\n      code === 0x200d ||\n      code === 0xfe0e ||\n      code === 0xfe0f\n    ) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/** snap the text offset to the Unicode boundary */\nexport function snapTextOffsetToUnicodeBoundary(\n  text: string,\n  offset: number\n): number {\n  const boundedOffset = Math.max(0, Math.min(offset, text.length));\n  if (\n    boundedOffset === 0 ||\n    boundedOffset === text.length ||\n    !needsDomTextMeasurement(text)\n  ) {\n    return boundedOffset;\n  }\n  // Avoid measuring a caret position inside one visual emoji/grapheme.\n  // Browser caret movement can report offsets around UTF-16 surrogate\n  // pairs and emoji joiners; measuring a partial sequence gives a\n  // replacement-glyph width.\n  const segmenter = getGraphemeSegmenter();\n  if (segmenter !== undefined) {\n    for (const segment of segmenter.segment(text)) {\n      const segmentStart = segment.index;\n      const segmentEnd = segmentStart + segment.segment.length;\n      if (boundedOffset > segmentStart && boundedOffset < segmentEnd) {\n        return segmentEnd;\n      }\n      if (boundedOffset <= segmentStart) {\n        break;\n      }\n    }\n    return boundedOffset;\n  }\n  // Degraded path for engines lacking Intl.Segmenter: snap out of a\n  // surrogate pair by stepping over code points.\n  let segmentStart = 0;\n  for (const codePoint of text) {\n    const segmentEnd = segmentStart + codePoint.length;\n    if (boundedOffset > segmentStart && boundedOffset < segmentEnd) {\n      return segmentEnd;\n    }\n    if (boundedOffset <= segmentStart) {\n      break;\n    }\n    segmentStart = segmentEnd;\n  }\n  return boundedOffset;\n}\n\n/** get the offsets of the Unicode grapheme clusters in the text */\nexport function getUnicodeMeasurementOffsets(\n  text: string\n): number[] | undefined {\n  if (!needsDomTextMeasurement(text)) {\n    return undefined;\n  }\n  const offsets = [0];\n  const segmenter = getGraphemeSegmenter();\n  if (segmenter !== undefined) {\n    for (const segment of segmenter.segment(text)) {\n      offsets.push(segment.index + segment.segment.length);\n    }\n    return offsets;\n  }\n  // Degraded path for engines lacking Intl.Segmenter: step by code point.\n  let offset = 0;\n  for (const codePoint of text) {\n    offset += codePoint.length;\n    offsets.push(offset);\n  }\n  return offsets;\n}\n\n/**\n * Expand tab characters to spaces using fixed tab stops: each tab advances to\n * the next multiple of tabSize from its running column, matching how the\n * rendered text expands tabs via CSS `tab-size`. Expanding every tab to a flat\n * tabSize would mis-measure tabs that follow other characters on the same line\n * (e.g. an alignment tab in `foo\\tbar`).\n */\nexport function expandTabsToSpaces(text: string, tabSize: number): string {\n  if (!text.includes('\\t')) {\n    return text;\n  }\n  let result = '';\n  let column = 0;\n  for (let i = 0; i < text.length; i++) {\n    if (text.charCodeAt(i) === /* '\\t' */ 9) {\n      const advance = tabSize - (column % tabSize);\n      result += ' '.repeat(advance);\n      column += advance;\n    } else {\n      result += text[i];\n      column += 1;\n    }\n  }\n  return result;\n}\n\n/**\n * Count the rendered columns of ASCII text, advancing each tab to the next\n * fixed tab stop (a multiple of tabSize) to match CSS `tab-size`. Returns -1\n * for non-ASCII text, which must be measured glyph-by-glyph instead.\n */\nexport function getExpandedAsciiTextColumns(\n  text: string,\n  tabSize: number\n): number {\n  let columns = 0;\n  for (let i = 0; i < text.length; i++) {\n    const code = text.charCodeAt(i);\n    if (code > 127) {\n      return -1;\n    }\n    columns += code === /* '\\t' */ 9 ? tabSize - (columns % tabSize) : 1;\n  }\n  return columns;\n}\n"],"mappings":";;AAMA,MAAM,yBAAyB;AAE/B,IAAa,UAAb,MAAqB;CACnB;CACA;CACA;CAGA,kCAAkB,IAAI,IAAoB;;CAG1C,KAAa;;CAEb,UAAkB;;CAElB,aAAqB;;CAErB,aAAqB;;CAGrB,KAAK,MAAyB;EAC5B,IACE,KAAKA,UAAU,QACf,KAAKC,eAAe,KAAA,KACpB,KAAK,OAAO,IAGZ;EAGF,KAAKD,QAAQ;EACb,KAAKC,eACH,SAAS,cAAc,QAAQ,CAAC,CAAC,WAAW,IAAI,KAAK,KAAA;EACvD,IAAI,KAAKA,eAAe,KAAA,GACtB,MAAM,IAAI,MAAM,8BAA8B;EAGhD,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,MAAM;GACnB,MAAM,EAAE,eAAe,iBAAiB,MAAM;GAC9C,IAAI,WAAW,SAAS,IAAI,GAC1B,KAAK,aAAa,WAAW,WAAW,MAAM,GAAG,EAAE,CAAC;EAExD;EAEA,MAAM,EAAE,UAAU,YAAY,SAAS,eACrC,iBAAiB,IAAI;EACvB,IAAI,WAAW,SAAS,IAAI,GAC1B,KAAK,aAAa,WAAW,WAAW,MAAM,GAAG,EAAE,CAAC;OAC/C,IAAI,SAAS,SAAS,IAAI,GAC/B,KAAK,aAAa,MAChB,WAAW,SAAS,MAAM,GAAG,EAAE,CAAC,IAAI,WAAW,UAAU,CAC3D;EAEF,MAAM,OAAO,WAAW,MAAM;EAC9B,IAAI,KAAKC,UAAU,QAAQ,KAAK,OAAO,IAAI;GACzC,KAAKA,QAAQ;GACb,KAAKD,WAAW,OAAO;GACvB,KAAK,KAAK,KAAK,uBAAuB,GAAG;GAEzC,KAAK,oBAAoB;EAC3B;EACA,MAAM,cAAc,SAAS,SAAS,EAAE;EACxC,IAAI,CAAC,OAAO,MAAM,WAAW,GAC3B,KAAK,UAAU;CAEnB;;;;;;;;;;;;;CAcA,0BAAmC;EACjC,IAAI,KAAKA,eAAe,KAAA,KAAa,KAAKC,UAAU,KAAA,GAClD,OAAO;EAET,KAAKD,WAAW,OAAO,KAAKC;EAC5B,MAAM,KAAK,KAAK,uBAAuB,GAAG;EAC1C,IAAI,OAAO,KAAK,IACd,OAAO;EAET,KAAK,KAAK;EACV,OAAO;CACT;;CAGA,iBAAiB,MAAsB;EACrC,MAAM,uBAAuB,mBAAmB,MAAM,KAAK,OAAO;EAClE,IAAI,wBAAwB,oBAAoB,GAC9C,OAAO,KAAK,oBAAoB,oBAAoB;EAEtD,OAAO,KAAK,uBAAuB,oBAAoB;CACzD;;CAGA,uBAAuB,MAAsB;EAC3C,IAAI,KAAKD,eAAe,KAAA,GACtB,MAAM,IAAI,MAAM,yBAAyB;EAE3C,OAAO,MAAM,KAAKA,WAAW,YAAY,IAAI,CAAC,CAAC,KAAK;CACtD;;;;;;CAOA,oBAAoB,MAAsB;EACxC,IAAI,KAAKD,UAAU,KAAA,GACjB,MAAM,IAAI,MAAM,yBAAyB;EAE3C,MAAM,WAAW,OAAO,MAAM,KAAKE;EACnC,MAAM,SAAS,KAAKC,gBAAgB,IAAI,QAAQ;EAChD,IAAI,WAAW,KAAA,GACb,OAAO;EAET,MAAM,YAAY,EAChB,QACA;GACE,OAAO;IACL,UAAU;IACV,KAAK;IACL,MAAM;IACN,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,MAAM;GACR;GACA,aAAa;EACf,GACA,KAAKH,KACP;EACA,IAAI;EACJ,IAAI;GAIF,QAAQ,MAAM,UAAU,sBAAsB,CAAC,CAAC,KAAK;EACvD,UAAU;GACR,UAAU,OAAO;EACnB;EACA,IAAI,KAAKG,gBAAgB,QAAQ,wBAAwB;GACvD,MAAM,YAAY,KAAKA,gBAAgB,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;GACrD,IAAI,cAAc,KAAA,GAChB,KAAKA,gBAAgB,OAAO,SAAS;EAEzC;EACA,KAAKA,gBAAgB,IAAI,UAAU,KAAK;EACxC,OAAO;CACT;;;;;;CAOA,sBAA4B;EAC1B,KAAKA,gBAAgB,MAAM;CAC7B;AACF;;AAGA,SAAgB,wBAAwB,MAAuB;CAC7D,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,OAAO,KAAK,WAAW,CAAC;EAC9B,IACG,QAAQ,SAAU,QAAQ,SAC3B,SAAS,QACT,SAAS,SACT,SAAS,OAET,OAAO;CAEX;CACA,OAAO;AACT;;AAGA,SAAgB,gCACd,MACA,QACQ;CACR,MAAM,gBAAgB,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,MAAM,CAAC;CAC/D,IACE,kBAAkB,KAClB,kBAAkB,KAAK,UACvB,CAAC,wBAAwB,IAAI,GAE7B,OAAO;CAMT,MAAM,YAAY,qBAAqB;CACvC,IAAI,cAAc,KAAA,GAAW;EAC3B,KAAK,MAAM,WAAW,UAAU,QAAQ,IAAI,GAAG;GAC7C,MAAM,eAAe,QAAQ;GAC7B,MAAM,aAAa,eAAe,QAAQ,QAAQ;GAClD,IAAI,gBAAgB,gBAAgB,gBAAgB,YAClD,OAAO;GAET,IAAI,iBAAiB,cACnB;EAEJ;EACA,OAAO;CACT;CAGA,IAAI,eAAe;CACnB,KAAK,MAAM,aAAa,MAAM;EAC5B,MAAM,aAAa,eAAe,UAAU;EAC5C,IAAI,gBAAgB,gBAAgB,gBAAgB,YAClD,OAAO;EAET,IAAI,iBAAiB,cACnB;EAEF,eAAe;CACjB;CACA,OAAO;AACT;;AAGA,SAAgB,6BACd,MACsB;CACtB,IAAI,CAAC,wBAAwB,IAAI,GAC/B;CAEF,MAAM,UAAU,CAAC,CAAC;CAClB,MAAM,YAAY,qBAAqB;CACvC,IAAI,cAAc,KAAA,GAAW;EAC3B,KAAK,MAAM,WAAW,UAAU,QAAQ,IAAI,GAC1C,QAAQ,KAAK,QAAQ,QAAQ,QAAQ,QAAQ,MAAM;EAErD,OAAO;CACT;CAEA,IAAI,SAAS;CACb,KAAK,MAAM,aAAa,MAAM;EAC5B,UAAU,UAAU;EACpB,QAAQ,KAAK,MAAM;CACrB;CACA,OAAO;AACT;;;;;;;;AASA,SAAgB,mBAAmB,MAAc,SAAyB;CACxE,IAAI,CAAC,KAAK,SAAS,GAAI,GACrB,OAAO;CAET,IAAI,SAAS;CACb,IAAI,SAAS;CACb,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAC/B,IAAI,KAAK,WAAW,CAAC,MAAiB,GAAG;EACvC,MAAM,UAAU,UAAW,SAAS;EACpC,UAAU,IAAI,OAAO,OAAO;EAC5B,UAAU;CACZ,OAAO;EACL,UAAU,KAAK;EACf,UAAU;CACZ;CAEF,OAAO;AACT;;;;;;AAOA,SAAgB,4BACd,MACA,SACQ;CACR,IAAI,UAAU;CACd,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,OAAO,KAAK,WAAW,CAAC;EAC9B,IAAI,OAAO,KACT,OAAO;EAET,WAAW,SAAoB,IAAI,UAAW,UAAU,UAAW;CACrE;CACA,OAAO;AACT"}

package/dist/editor/tokenzier.d.ts

@@ -9,6 +9,7 @@ interface EditorTokenizerProps {
   codeOptions: BaseCodeOptions;
   setStyle: (style: string) => void;
   onDeferTokenize: (lines: Map<number, Array<HighlightedToken>>, themeType: 'dark' | 'light') => void;
+  onThemeChange?: () => void;
   __debug?: boolean;
 }
 /** Stoppable code tokenizer for the editor */
@@ -22,8 +23,10 @@ declare class EditorTokenizer {
     textDocument,
     setStyle,
     onDeferTokenize,
+    onThemeChange,
     __debug
   }: EditorTokenizerProps);
+  syncTheme(codeOptions: BaseCodeOptions): void;
   cleanUp(): void;
   tokenize(change: TextDocumentChange, renderRange?: RenderRange): Map<number, Array<HighlightedToken>>;
   prebuildStateStack(renderRange?: RenderRange): void;

package/dist/editor/tokenzier.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenzier.d.ts","names":["IGrammar","StateStack","BaseCodeOptions","DiffsHighlighter","HighlightedToken","RenderRange","TextDocument","TextDocumentChange","EditorTokenizerProps","Array","Map","EditorTokenizer","codeOptions","highlighter","textDocument","setStyle","onDeferTokenize","__debug","tokenizeLine","renderLineTokens","HTMLElement"],"sources":["../../src/editor/tokenzier.d.ts"],"sourcesContent":["import { type IGrammar, type StateStack } from 'shiki/textmate';\nimport type { BaseCodeOptions, DiffsHighlighter, HighlightedToken, RenderRange } from '../types';\nimport type { TextDocument, TextDocumentChange } from './textDocument';\nexport interface EditorTokenizerProps {\n    highlighter: DiffsHighlighter;\n    textDocument: TextDocument<unknown>;\n    codeOptions: BaseCodeOptions;\n    setStyle: (style: string) => void;\n    onDeferTokenize: (lines: Map<number, Array<HighlightedToken>>, themeType: 'dark' | 'light') => void;\n    __debug?: boolean;\n}\n/** Stoppable code tokenizer for the editor */\nexport declare class EditorTokenizer {\n    #private;\n    static TOKENIZE_TIME_LIMIT: number;\n    get themeType(): 'light' | 'dark';\n    constructor({ codeOptions, highlighter, textDocument, setStyle, onDeferTokenize, __debug }: EditorTokenizerProps);\n    cleanUp(): void;\n    tokenize(change: TextDocumentChange, renderRange?: RenderRange): Map<number, Array<HighlightedToken>>;\n    prebuildStateStack(renderRange?: RenderRange): void;\n    stopBackgroundTokenize(): void;\n    pauseBackgroundTokenize(): void;\n    resumeBackgroundTokenize(): void;\n}\nexport declare function tokenizeLine(grammar: IGrammar, colorMap: string[], lineText: string, stateStack: StateStack, timeLimit?: number): {\n    ruleStack: StateStack;\n    resolvedTokens: Array<HighlightedToken>;\n};\nexport declare function renderLineTokens(tokens: Array<HighlightedToken>, themeType: 'light' | 'dark'): (HTMLElement | string)[];\n//# sourceMappingURL=tokenzier.d.ts.map"],"mappings":";;;;;UAGiBQ,oBAAAA;eACAL;EADAK,YAAAA,EAECF,YAFmB,CAAA,OAAA,CAAA;EACpBH,WAAAA,EAEAD,eAFAC;EACCG,QAAAA,EAAAA,CAAAA,KAAAA,EAAAA,MAAAA,EAAAA,GAAAA,IAAAA;EACDJ,eAAAA,EAAAA,CAAAA,KAAAA,EAEYQ,GAFZR,CAAAA,MAAAA,EAEwBO,KAFxBP,CAE8BE,gBAF9BF,CAAAA,CAAAA,EAAAA,SAAAA,EAAAA,MAAAA,GAAAA,OAAAA,EAAAA,GAAAA,IAAAA;EAE8BE,OAAAA,CAAAA,EAAAA,OAAAA;;;AAAf,cAIXO,eAAAA,CAJW;EAIXA,CAAAA,OAAAA;EAIHC,OAAAA,mBAAAA,EAAAA,MAAAA;EAAaC,IAAAA,SAAAA,CAAAA,CAAAA,EAAAA,OAAAA,GAAAA,MAAAA;EAAaC,WAAAA,CAAAA;IAAAA,WAAAA;IAAAA,WAAAA;IAAAA,YAAAA;IAAAA,QAAAA;IAAAA,eAAAA;IAAAA;EAAAA,CAAAA,EAAoDN,oBAApDM;EAAcC,OAAAA,CAAAA,CAAAA,EAAAA,IAAAA;EAAUC,QAAAA,CAAAA,MAAAA,EAE/CT,kBAF+CS,EAAAA,WAAAA,CAAAA,EAEbX,WAFaW,CAAAA,EAECN,GAFDM,CAAAA,MAAAA,EAEaP,KAFbO,CAEmBZ,gBAFnBY,CAAAA,CAAAA;EAAiBC,kBAAAA,CAAAA,WAAAA,CAAAA,EAGhDZ,WAHgDY,CAAAA,EAAAA,IAAAA;EAAWT,sBAAAA,CAAAA,CAAAA,EAAAA,IAAAA;EAE3ED,uBAAAA,CAAAA,CAAAA,EAAAA,IAAAA;EAAkCF,wBAAAA,CAAAA,CAAAA,EAAAA,IAAAA;;AAA0BI,iBAMzDS,YAAAA,CANyDT,OAAAA,EAMnCT,QANmCS,EAAAA,QAAAA,EAAAA,MAAAA,EAAAA,EAAAA,QAAAA,EAAAA,MAAAA,EAAAA,UAAAA,EAMyBR,UANzBQ,EAAAA,SAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA;EAAZC,SAAAA,EAOtDT,UAPsDS;EAChCL,cAAAA,EAOjBI,KAPiBJ,CAOXD,gBAPWC,CAAAA;CAAW;AAKxBa,iBAIAC,gBAAAA,CAJY,MAAA,EAIaV,KAJb,CAImBL,gBAJnB,CAAA,EAAA,SAAA,EAAA,OAAA,GAAA,MAAA,CAAA,EAAA,CAIqEgB,WAJrE,GAAA,MAAA,CAAA,EAAA"}
+{"version":3,"file":"tokenzier.d.ts","names":[],"sources":["../../src/editor/tokenzier.ts"],"mappings":";;;;;UAiBiB,oBAAA;EACf,WAAA,EAAa,gBAAA;EACb,YAAA,EAAc,YAAA;EACd,WAAA,EAAa,eAAA;EACb,QAAA,GAAW,KAAA;EACX,eAAA,GACE,KAAA,EAAO,GAAA,SAAY,KAAA,CAAM,gBAAA,IACzB,SAAA;EAKF,aAAA;EACA,OAAA;AAAA;;cAIW,eAAA;EAAA;SACJ,mBAAA;EAAA,IAgEH,SAAA;EAIJ,WAAA;IACE,WAAA;IACA,WAAA;IACA,YAAA;IACA,QAAA;IACA,eAAA;IACA,aAAA;IACA;EAAA,GACC,oBAAA;EAgFH,SAAA,CAAU,WAAA,EAAa,eAAA;EA8CvB,OAAA;EASA,QAAA,CACE,MAAA,EAAQ,kBAAA,EACR,WAAA,GAAc,WAAA,GACb,GAAA,SAAY,KAAA,CAAM,gBAAA;EA0LrB,kBAAA,CAAmB,WAAA,GAAc,WAAA;EAmBjC,sBAAA;EAYA,uBAAA;EAYA,wBAAA;AAAA;AAAA,iBA4Nc,YAAA,CACd,OAAA,EAAS,QAAA,EACT,QAAA,YACA,QAAA,UACA,UAAA,EAAY,UAAA,EACZ,SAAA;EAEA,SAAA,EAAW,UAAA;EACX,cAAA,EAAgB,KAAA,CAAM,gBAAA;AAAA;AAAA,iBA+BR,gBAAA,CACd,MAAA,EAAQ,KAAA,CAAM,gBAAA,GACd,SAAA,sBACE,WAAA"}

package/dist/editor/tokenzier.js

@@ -1,7 +1,6 @@
 import { DEFAULT_THEMES } from "../constants.js";
 import { addEventListener, debounce, h } from "./utils.js";
 import { EncodedTokenMetadata, INITIAL } from "shiki/textmate";
-
 //#region src/editor/tokenzier.ts
 /** Stoppable code tokenizer for the editor */
 var EditorTokenizer = class EditorTokenizer {
@@ -10,11 +9,13 @@ var EditorTokenizer = class EditorTokenizer {
 	#grammar;
 	#mediaQueryList;
 	#themeType;
+	#themeName = "";
 	#colorMap;
 	#textDocument;
 	#tokenizeMaxLineLength;
 	#setStyle;
 	#onDeferTokenize;
+	#onThemeChange;
 	#debug;
 	#disposes;
 	#stateStack = [INITIAL];
@@ -42,24 +43,24 @@ var EditorTokenizer = class EditorTokenizer {
 	get themeType() {
 		return this.#themeType;
 	}
-	constructor({ codeOptions, highlighter, textDocument, setStyle, onDeferTokenize, __debug }) {
+	constructor({ codeOptions, highlighter, textDocument, setStyle, onDeferTokenize, onThemeChange, __debug }) {
 		const { themeType = "system", theme = DEFAULT_THEMES, tokenizeMaxLineLength = 1e3 } = codeOptions;
 		this.#mediaQueryList = window.matchMedia("(prefers-color-scheme: dark)");
 		if (themeType === "system") this.#themeType = this.#mediaQueryList.matches ? "dark" : "light";
 		else this.#themeType = themeType;
-		if (typeof theme !== "string") {
+		if (typeof theme !== "string" && themeType === "system") {
 			const observer = new MutationObserver((mutations) => {
 				for (const { type, attributeName } of mutations) if (type === "attributes" && attributeName !== null && (attributeName === "class" || attributeName.startsWith("data-"))) {
-					const themeType$1 = getComputedStyle(document.body).colorScheme === "dark" ? "dark" : "light";
-					this.#emitThemeChange(theme[themeType$1], themeType$1);
+					const themeType = getComputedStyle(document.body).colorScheme === "dark" ? "dark" : "light";
+					this.#emitThemeChange(theme[themeType], themeType);
 					break;
 				}
 			});
 			observer.observe(document.documentElement, { attributes: true });
 			observer.observe(document.body, { attributes: true });
 			this.#disposes = [addEventListener(this.#mediaQueryList, "change", (e) => {
-				const themeType$1 = e.matches ? "dark" : "light";
-				this.#emitThemeChange(theme[themeType$1], themeType$1);
+				const themeType = e.matches ? "dark" : "light";
+				this.#emitThemeChange(theme[themeType], themeType);
 			}), () => observer.disconnect()];
 		}
 		this.#highlighter = highlighter;
@@ -67,8 +68,9 @@ var EditorTokenizer = class EditorTokenizer {
 		this.#tokenizeMaxLineLength = tokenizeMaxLineLength;
 		this.#setStyle = setStyle;
 		this.#onDeferTokenize = onDeferTokenize;
+		this.#onThemeChange = onThemeChange;
 		this.#debug = __debug ?? false;
-		if (!isGrammarlessLanguage(textDocument.languageId) && highlighter.getLoadedLanguages().includes(textDocument.languageId)) this.#grammar = highlighter.getLanguage(textDocument.languageId);
+		this.#ensureGrammar();
 		this.#colorMap = [];
 		this.#setTheme(typeof theme === "string" ? theme : theme[this.#themeType]);
 	}
@@ -78,14 +80,21 @@ var EditorTokenizer = class EditorTokenizer {
 		this.stopBackgroundTokenize();
 		this.#stateStack = [INITIAL];
 		if (this.#grammar !== void 0 && this.#textDocument.lineCount > 0) this.#scheduleBackgroundTokenize(0);
+		this.#onThemeChange?.();
+	}
+	syncTheme(codeOptions) {
+		const { themeType = "system", theme = DEFAULT_THEMES } = codeOptions;
+		const nextThemeType = themeType === "system" ? this.#mediaQueryList.matches ? "dark" : "light" : themeType;
+		const nextThemeName = typeof theme === "string" ? theme : theme[nextThemeType];
+		if (nextThemeType === this.#themeType && nextThemeName === this.#themeName) return;
+		this.#emitThemeChange(nextThemeName, nextThemeType);
 	}
 	#setTheme(themeName) {
+		this.#themeName = themeName;
 		this.#colorMap = this.#highlighter.setTheme(themeName).colorMap;
 		const { colors = {} } = this.#highlighter.getTheme(themeName);
 		const selectionBackground = colors["editor.selectionBackground"];
 		const lineHighlightBackground = colors["editor.lineHighlightBackground"];
-		const gutterForeground = colors["editorLineNumber.foreground"];
-		const gutterActiveForeground = colors["editorLineNumber.activeForeground"];
 		const cursorForeground = colors["editorCursor.foreground"];
 		const findMatchBackground = colors["editor.findMatchBackground"];
 		const findMatchHighlightBackground = colors["editor.findMatchHighlightBackground"];
@@ -96,9 +105,6 @@ var EditorTokenizer = class EditorTokenizer {
 		this.#setStyle(`:host {
       --diffs-editor-selection-bg: ${selectionBackground ?? "var(--diffs-line-bg)"};
       --diffs-editor-line-highlight-bg: ${lineHighlightBackground ?? "var(--diffs-line-bg)"};
-      --diffs-editor-line-number-fg: ${gutterForeground ?? "var(--diffs-fg-number)"};
-      --diffs-editor-line-number-active-bg: ${lineHighlightBackground ?? "var(--diffs-line-bg, var(--diffs-bg))"};
-      --diffs-editor-line-number-active-fg: ${gutterActiveForeground ?? "var(--diffs-selection-number-fg)"};
       --diffs-editor-match-bg: ${findMatchBackground ?? "unset"};
       --diffs-editor-match-highlight-bg: ${findMatchHighlightBackground ?? "unset"};
       --diffs-editor-cursor-fg: ${cursorForeground ?? "unset"};
@@ -115,7 +121,8 @@ var EditorTokenizer = class EditorTokenizer {
 		this.#disposes = void 0;
 	}
 	tokenize(change, renderRange) {
-		if (this.#grammar === void 0 && !isGrammarlessLanguage(this.#textDocument.languageId)) throw new Error("Grammar not loaded");
+		this.#ensureGrammar();
+		if (this.#grammar === void 0 && !isGrammarlessLanguage(this.#textDocument.languageId)) throw new Error(`Grammar for language "${this.#textDocument.languageId}" not loaded`);
 		const { lineCount } = this.#textDocument;
 		const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};
 		const renderRangeEndLine = totalLines === Infinity ? lineCount : Math.min(startingLine + totalLines, lineCount);
@@ -140,7 +147,6 @@ var EditorTokenizer = class EditorTokenizer {
 		let backgroundStartLine;
 		let backgroundChangedRangeIndex = 0;
 		let line = canReuseCachedStates ? changedLineRanges[changedRangeIndex][0] : viewStart;
-		let state = this.#stateStack[line] ?? INITIAL;
 		let settled = false;
 		const dirtyLines = /* @__PURE__ */ new Map();
 		const offscreenDirtyLines = shouldFlushOffscreenLines ? /* @__PURE__ */ new Map() : void 0;
@@ -158,6 +164,7 @@ var EditorTokenizer = class EditorTokenizer {
 				if (canCacheTokenizedStates) this.#stateStack[offscreenEnd] = offscreenState;
 			}
 		}
+		let state = this.#stateStack[line] ?? INITIAL;
 		for (; line < renderRangeEndLine;) {
 			const previousNextState = canReuseCachedStates ? this.#stateStack[line + 1] : void 0;
 			if (canCacheTokenizedStates) this.#stateStack[line] = state;
@@ -200,8 +207,12 @@ var EditorTokenizer = class EditorTokenizer {
 		return dirtyLines;
 	}
 	prebuildStateStack(renderRange) {
+		this.#ensureGrammar();
 		this.#prebuildStateStack(renderRange);
 	}
+	#ensureGrammar() {
+		if (this.#grammar === void 0 && !isGrammarlessLanguage(this.#textDocument.languageId) && this.#highlighter.getLoadedLanguages().includes(this.#textDocument.languageId)) this.#grammar = this.#highlighter.getLanguage(this.#textDocument.languageId);
+	}
 	stopBackgroundTokenize() {
 		if (this.#isStopped) return;
 		this.#isStopped = true;
@@ -395,7 +406,7 @@ function renderLineTokens(tokens, themeType) {
 function isGrammarlessLanguage(languageId) {
 	return languageId === "text" || languageId === "ansi";
 }
-
 //#endregion
 export { EditorTokenizer, renderLineTokens, tokenizeLine };
+
 //# sourceMappingURL=tokenzier.js.map

package/dist/editor/tokenzier.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenzier.js","names":["#textDocument","#grammar","#highlighter","#buildStateStack","#backgroundJobId","#backgroundTokenize","#themeType","#mediaQueryList","themeType","#emitThemeChange","#disposes","#tokenizeMaxLineLength","#setStyle","#onDeferTokenize","#debug","#colorMap","#setTheme","#stateStack","#scheduleBackgroundTokenize","#detachMessageListener","changedLineRanges: readonly [number, number][]","backgroundStartLine: number | undefined","dirtyLines: Map<number, Array<HighlightedToken>>","offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined","#tokenizeLineAt","#prebuildStateStack","#isStopped","#isPaused","#lastLine","#backgroundChangedLineRanges","#backgroundChangedRangeIndex","#postTokenizeMessage","#isMessageListenerAttached","#onMessage","#attachMessageListener","resolvedTokens: Array<HighlightedToken>"],"sources":["../../src/editor/tokenzier.ts"],"sourcesContent":["import {\n  EncodedTokenMetadata,\n  type IGrammar,\n  INITIAL,\n  type StateStack,\n} from 'shiki/textmate';\n\nimport { DEFAULT_THEMES } from '../constants';\nimport type {\n  BaseCodeOptions,\n  DiffsHighlighter,\n  HighlightedToken,\n  RenderRange,\n} from '../types';\nimport type { TextDocument, TextDocumentChange } from './textDocument';\nimport { addEventListener, debounce, h } from './utils';\n\nexport interface EditorTokenizerProps {\n  highlighter: DiffsHighlighter;\n  textDocument: TextDocument<unknown>;\n  codeOptions: BaseCodeOptions;\n  setStyle: (style: string) => void;\n  onDeferTokenize: (\n    lines: Map<number, Array<HighlightedToken>>,\n    themeType: 'dark' | 'light'\n  ) => void;\n  __debug?: boolean;\n}\n\n/** Stoppable code tokenizer for the editor */\nexport class EditorTokenizer {\n  static TOKENIZE_TIME_LIMIT = 500;\n\n  #highlighter: DiffsHighlighter;\n  #grammar: IGrammar | undefined;\n  #mediaQueryList: MediaQueryList;\n  #themeType: 'light' | 'dark';\n  #colorMap: string[];\n  #textDocument: TextDocument<unknown>;\n  #tokenizeMaxLineLength: number;\n  #setStyle: EditorTokenizerProps['setStyle'];\n  #onDeferTokenize: EditorTokenizerProps['onDeferTokenize'];\n  #debug: boolean;\n  #disposes?: (() => void)[];\n\n  // state\n  #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index\n  #lastLine: number = -1;\n  #isStopped: boolean = true;\n  #isPaused: boolean = false;\n  #backgroundJobId: number = 0;\n  #backgroundChangedLineRanges: readonly [number, number][] | undefined;\n  #backgroundChangedRangeIndex: number = 0;\n  #isMessageListenerAttached: boolean = false;\n\n  #prebuildStateStack = debounce(async (renderRange?: RenderRange) => {\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const endLine = Math.min(\n      totalLines === Infinity ? Infinity : startingLine + totalLines,\n      this.#textDocument.lineCount\n    );\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      await this.#highlighter.loadLanguage(this.#textDocument.languageId);\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n    this.#buildStateStack(endLine);\n  }, 500);\n\n  #onMessage = ({ data }: MessageEvent<unknown>) => {\n    if (typeof data !== 'object' || data === null) {\n      return;\n    }\n    const { type, jobId } = data as {\n      type?: unknown;\n      jobId?: unknown;\n    };\n    if (\n      type === 'tokenize' &&\n      typeof jobId === 'number' &&\n      jobId === this.#backgroundJobId\n    ) {\n      this.#backgroundTokenize(jobId);\n    }\n  };\n\n  get themeType(): 'light' | 'dark' {\n    return this.#themeType;\n  }\n\n  constructor({\n    codeOptions,\n    highlighter,\n    textDocument,\n    setStyle,\n    onDeferTokenize,\n    __debug,\n  }: EditorTokenizerProps) {\n    const {\n      themeType = 'system',\n      theme = DEFAULT_THEMES,\n      tokenizeMaxLineLength = 1000,\n    } = codeOptions;\n    this.#mediaQueryList = window.matchMedia('(prefers-color-scheme: dark)');\n    if (themeType === 'system') {\n      this.#themeType = this.#mediaQueryList.matches ? 'dark' : 'light';\n    } else {\n      this.#themeType = themeType;\n    }\n    if (typeof theme !== 'string') {\n      const observer = new MutationObserver((mutations) => {\n        for (const { type, attributeName } of mutations) {\n          if (\n            type === 'attributes' &&\n            attributeName !== null &&\n            (attributeName === 'class' || attributeName.startsWith('data-'))\n          ) {\n            const themeType =\n              getComputedStyle(document.body).colorScheme === 'dark'\n                ? 'dark'\n                : 'light';\n            this.#emitThemeChange(theme[themeType], themeType);\n            break;\n          }\n        }\n      });\n      observer.observe(document.documentElement, { attributes: true });\n      observer.observe(document.body, { attributes: true });\n      this.#disposes = [\n        addEventListener(this.#mediaQueryList, 'change', (e) => {\n          const themeType = e.matches ? 'dark' : 'light';\n          this.#emitThemeChange(theme[themeType], themeType);\n        }),\n        () => observer.disconnect(),\n      ];\n    }\n    this.#highlighter = highlighter;\n    this.#textDocument = textDocument;\n    this.#tokenizeMaxLineLength = tokenizeMaxLineLength;\n    this.#setStyle = setStyle;\n    this.#onDeferTokenize = onDeferTokenize;\n    this.#debug = __debug ?? false;\n    if (\n      !isGrammarlessLanguage(textDocument.languageId) &&\n      highlighter.getLoadedLanguages().includes(textDocument.languageId)\n    ) {\n      this.#grammar = highlighter.getLanguage(textDocument.languageId);\n    }\n    this.#colorMap = [];\n    this.#setTheme(typeof theme === 'string' ? theme : theme[this.#themeType]);\n  }\n\n  // By default, diffs components support dual themes, but the tokenizer only renders\n  // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document.\n  #emitThemeChange(themeName: string, themeType: 'light' | 'dark') {\n    this.#themeType = themeType;\n    this.#setTheme(themeName);\n    this.stopBackgroundTokenize();\n    this.#stateStack = [INITIAL];\n    if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) {\n      this.#scheduleBackgroundTokenize(0);\n    }\n  }\n\n  #setTheme(themeName: string) {\n    this.#colorMap = this.#highlighter.setTheme(themeName).colorMap;\n    const { colors = {} } = this.#highlighter.getTheme(themeName);\n    const selectionBackground = colors['editor.selectionBackground'];\n    const lineHighlightBackground = colors['editor.lineHighlightBackground'];\n    const gutterForeground = colors['editorLineNumber.foreground'];\n    const gutterActiveForeground = colors['editorLineNumber.activeForeground'];\n    const cursorForeground = colors['editorCursor.foreground'];\n    const findMatchBackground = colors['editor.findMatchBackground'];\n    const findMatchHighlightBackground =\n      colors['editor.findMatchHighlightBackground'];\n    const hintForeground = colors['editorHint.foreground'];\n    const infoForeground = colors['editorInfo.foreground'];\n    const warningForeground = colors['editorWarning.foreground'];\n    const errorForeground = colors['editorError.foreground'];\n    this.#setStyle(`:host {\n      --diffs-editor-selection-bg: ${selectionBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-highlight-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-number-fg: ${gutterForeground ?? 'var(--diffs-fg-number)'};\n      --diffs-editor-line-number-active-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg, var(--diffs-bg))'};\n      --diffs-editor-line-number-active-fg: ${gutterActiveForeground ?? 'var(--diffs-selection-number-fg)'};\n      --diffs-editor-match-bg: ${findMatchBackground ?? 'unset'};\n      --diffs-editor-match-highlight-bg: ${findMatchHighlightBackground ?? 'unset'};\n      --diffs-editor-cursor-fg: ${cursorForeground ?? 'unset'};\n      --diffs-editor-hint-fg: ${hintForeground ?? 'unset'};\n      --diffs-editor-info-fg: ${infoForeground ?? 'unset'};\n      --diffs-editor-warning-fg: ${warningForeground ?? 'unset'};\n      --diffs-editor-error-fg: ${errorForeground ?? 'unset'};\n    }`);\n  }\n\n  cleanUp(): void {\n    this.stopBackgroundTokenize();\n    this.#detachMessageListener();\n    this.#disposes?.forEach((dispose) => dispose());\n    this.#disposes = undefined;\n  }\n\n  // to use `tokenize`, call `prebuildStateStackMap` first to prebuild\n  // the state stack map for the given render range.\n  tokenize(\n    change: TextDocumentChange,\n    renderRange?: RenderRange\n  ): Map<number, Array<HighlightedToken>> {\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      throw new Error('Grammar not loaded');\n    }\n\n    const { lineCount } = this.#textDocument;\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const renderRangeEndLine =\n      totalLines === Infinity\n        ? lineCount\n        : Math.min(startingLine + totalLines, lineCount);\n\n    const dirtyStart = change.startLine;\n    const viewStart = Math.max(startingLine, dirtyStart);\n    const crossesRenderRangeEnd =\n      renderRange !== undefined &&\n      totalLines !== Infinity &&\n      change.lineDelta > 0 &&\n      dirtyStart < renderRangeEndLine &&\n      change.endLine >= renderRangeEndLine;\n    const canReuseCachedStates = change.lineDelta === 0;\n    const canCacheTokenizedStates =\n      canReuseCachedStates ||\n      renderRange === undefined ||\n      dirtyStart >= viewStart;\n    const changedLineRanges: readonly [number, number][] = canReuseCachedStates\n      ? (change.changedLineRanges ?? [[dirtyStart, change.endLine]])\n      : [[dirtyStart, change.endLine]];\n    let offscreenSyncEnd = -1;\n    if (dirtyStart < viewStart) {\n      for (const [rangeStart, rangeEnd] of changedLineRanges) {\n        if (rangeStart < viewStart) {\n          offscreenSyncEnd = Math.max(\n            offscreenSyncEnd,\n            Math.min(rangeEnd, viewStart - 1)\n          );\n        }\n      }\n    }\n    const shouldFlushOffscreenLines =\n      offscreenSyncEnd >= dirtyStart &&\n      (canReuseCachedStates || change.lineDelta < 0);\n    if (canReuseCachedStates) {\n      this.#buildStateStack(dirtyStart);\n    } else {\n      this.#stateStack.length = Math.min(\n        this.#stateStack.length,\n        dirtyStart + 1\n      );\n      if (renderRange === undefined || dirtyStart >= viewStart) {\n        this.#buildStateStack(viewStart);\n      }\n    }\n\n    let changedRangeIndex = 0;\n    let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];\n    let backgroundStartLine: number | undefined;\n    let backgroundChangedRangeIndex = 0;\n    let line = canReuseCachedStates\n      ? changedLineRanges[changedRangeIndex][0]\n      : viewStart;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    const dirtyLines: Map<number, Array<HighlightedToken>> = new Map();\n    const offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined = shouldFlushOffscreenLines ? new Map() : undefined;\n    if (offscreenDirtyLines !== undefined && !canReuseCachedStates) {\n      const offscreenEnd = Math.min(\n        offscreenSyncEnd + 1,\n        viewStart,\n        renderRangeEndLine\n      );\n      if (offscreenEnd > dirtyStart) {\n        this.#buildStateStack(offscreenEnd);\n        let offscreenLine = dirtyStart;\n        let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;\n        for (; offscreenLine < offscreenEnd; offscreenLine++) {\n          const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);\n          offscreenState = resolved.state;\n          offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);\n        }\n        if (canCacheTokenizedStates) {\n          this.#stateStack[offscreenEnd] = offscreenState;\n        }\n      }\n    }\n    for (; line < renderRangeEndLine; ) {\n      const previousNextState = canReuseCachedStates\n        ? this.#stateStack[line + 1]\n        : undefined;\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line] = state;\n      }\n\n      const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(\n        line,\n        state\n      );\n      state = nextState;\n\n      if (line >= viewStart) {\n        dirtyLines.set(line, resolvedTokens);\n      } else {\n        offscreenDirtyLines?.set(line, resolvedTokens);\n      }\n\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line + 1] = state;\n      }\n      settled =\n        line >= currentChangedRangeEnd &&\n        canReuseCachedStates &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        if (nextRange[0] >= renderRangeEndLine) {\n          backgroundStartLine = nextRange[0];\n          backgroundChangedRangeIndex = changedRangeIndex;\n          break;\n        }\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          currentChangedRangeEnd = nextRange[1];\n          line++;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          currentChangedRangeEnd = nextRange[1];\n        }\n        settled = false;\n        continue;\n      }\n      line++;\n    }\n\n    if (canCacheTokenizedStates) {\n      if (line < renderRangeEndLine) {\n        this.#stateStack[line + 1] = state;\n      } else {\n        this.#stateStack[line] = state;\n      }\n    }\n\n    if (offscreenDirtyLines !== undefined && offscreenDirtyLines.size > 0) {\n      this.#onDeferTokenize(offscreenDirtyLines, this.#themeType);\n    }\n\n    if (backgroundStartLine !== undefined) {\n      this.#scheduleBackgroundTokenize(\n        backgroundStartLine,\n        changedLineRanges,\n        backgroundChangedRangeIndex\n      );\n    } else if (!settled && line < lineCount) {\n      const backgroundLine =\n        crossesRenderRangeEnd && dirtyStart >= viewStart\n          ? renderRangeEndLine\n          : dirtyStart < viewStart && !canReuseCachedStates\n            ? dirtyStart\n            : line;\n      this.#scheduleBackgroundTokenize(\n        backgroundLine,\n        canReuseCachedStates ? changedLineRanges : undefined,\n        changedRangeIndex\n      );\n    }\n\n    return dirtyLines;\n  }\n\n  prebuildStateStack(renderRange?: RenderRange): void {\n    this.#prebuildStateStack(renderRange);\n  }\n\n  stopBackgroundTokenize(): void {\n    if (this.#isStopped) {\n      return;\n    }\n    this.#isStopped = true;\n    this.#isPaused = false;\n    this.#lastLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#detachMessageListener();\n  }\n\n  pauseBackgroundTokenize(): void {\n    if (this.#isStopped || this.#isPaused) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization paused', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = true;\n  }\n\n  resumeBackgroundTokenize(): void {\n    if (\n      this.#isStopped ||\n      !this.#isPaused ||\n      this.#grammar === undefined ||\n      this.#lastLine < 0\n    ) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization resumed', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = false;\n    this.#postTokenizeMessage(this.#backgroundJobId);\n  }\n\n  #attachMessageListener(): void {\n    if (this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.addEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = true;\n  }\n\n  #detachMessageListener(): void {\n    if (!this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.removeEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = false;\n  }\n\n  #postTokenizeMessage(jobId: number): void {\n    // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay\n    globalThis.postMessage({ type: 'tokenize', jobId });\n  }\n\n  #scheduleBackgroundTokenize(\n    startLine: number,\n    changedLineRanges?: readonly [number, number][],\n    changedRangeIndex = 0\n  ): void {\n    if (isGrammarlessLanguage(this.#textDocument.languageId)) {\n      return;\n    }\n\n    const jobId = ++this.#backgroundJobId;\n\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization scheduled', {\n        jobId,\n        startLine,\n        changedLineRanges,\n        changedRangeIndex,\n      });\n    }\n\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = startLine;\n    this.#backgroundChangedLineRanges = changedLineRanges;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #tokenizeLineAt(\n    line: number,\n    state: StateStack\n  ): { resolvedTokens: Array<HighlightedToken>; state: StateStack } {\n    const lineText = this.#textDocument.getLineText(line);\n    if (lineText.length > this.#tokenizeMaxLineLength) {\n      console.warn(\n        `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n      );\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    if (\n      this.#grammar === undefined ||\n      lineText === '' ||\n      lineText.trim() === ''\n    ) {\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    const result = tokenizeLine(\n      this.#grammar,\n      this.#colorMap,\n      lineText,\n      state,\n      EditorTokenizer.TOKENIZE_TIME_LIMIT\n    );\n    return {\n      resolvedTokens: result.resolvedTokens,\n      state: result.ruleStack,\n    };\n  }\n\n  #buildStateStack(endAt: number) {\n    const boundedEndAt = Math.min(\n      Math.max(0, endAt),\n      this.#textDocument.lineCount\n    );\n    if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) {\n      return;\n    }\n    let line = this.#stateStack.length - 1;\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < boundedEndAt; line++) {\n      this.#stateStack[line] = state;\n      const lineText = this.#textDocument.getLineText(line);\n      if (\n        lineText.length <= this.#tokenizeMaxLineLength &&\n        lineText !== '' &&\n        lineText.trim() !== ''\n      ) {\n        state = this.#grammar.tokenizeLine2(\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        ).ruleStack;\n      }\n    }\n    this.#stateStack[line] = state;\n  }\n\n  #backgroundTokenize(jobId: number) {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar === undefined ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    const t = performance.now();\n    const lines = new Map<number, Array<HighlightedToken>>();\n    const totalLines = this.#textDocument.lineCount;\n    const changedLineRanges = this.#backgroundChangedLineRanges;\n\n    let line = this.#lastLine;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    let changedRangeIndex = this.#backgroundChangedRangeIndex;\n    let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];\n    for (; line < totalLines; ) {\n      this.#stateStack[line] = state;\n\n      const previousNextState =\n        currentChangedRangeEnd !== undefined\n          ? this.#stateStack[line + 1]\n          : undefined;\n      const lineText = this.#textDocument.getLineText(line);\n      if (lineText.length > this.#tokenizeMaxLineLength) {\n        console.warn(\n          `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n        );\n        lines.set(line, [[0, '', lineText]]);\n      } else if (lineText === '' || lineText.trim() === '') {\n        lines.set(line, [[0, '', lineText]]);\n      } else {\n        const ret = tokenizeLine(\n          this.#grammar,\n          this.#colorMap,\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        );\n        lines.set(line, ret.resolvedTokens);\n        state = ret.ruleStack;\n      }\n\n      this.#stateStack[line + 1] = state;\n      settled =\n        currentChangedRangeEnd !== undefined &&\n        line >= currentChangedRangeEnd &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      line++;\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges?.[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        currentChangedRangeEnd = nextRange[1];\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          settled = false;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          settled = false;\n          continue;\n        }\n      }\n\n      // limit the time of partial tokenize to 1ms\n      if (performance.now() - t > 1) {\n        break;\n      }\n    }\n\n    this.#onDeferTokenize(lines, this.#themeType);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n\n    if (settled || line >= totalLines) {\n      this.stopBackgroundTokenize();\n      return;\n    }\n\n    this.#lastLine = line;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#postTokenizeMessage(jobId);\n  }\n}\n\nexport function tokenizeLine(\n  grammar: IGrammar,\n  colorMap: string[],\n  lineText: string,\n  stateStack: StateStack,\n  timeLimit?: number\n): {\n  ruleStack: StateStack;\n  resolvedTokens: Array<HighlightedToken>;\n} {\n  const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);\n  if (result.stoppedEarly) {\n    console.warn(\n      `[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`\n    );\n  }\n  const rawTokens = result.tokens;\n  const tokensLength = rawTokens.length / 2;\n  const resolvedTokens: Array<HighlightedToken> = [];\n  for (let j = 0; j < tokensLength; j++) {\n    const offset = rawTokens[2 * j];\n    const nextOffset =\n      j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;\n    if (offset === nextOffset) {\n      // should never reach here, skip if happens anyway\n      continue;\n    }\n    const metadata = rawTokens[2 * j + 1];\n    const bg = EncodedTokenMetadata.getForeground(metadata);\n    const fg = colorMap[bg];\n    const tokenText = lineText.slice(offset, nextOffset);\n    resolvedTokens.push([offset, fg, tokenText]);\n  }\n  return {\n    ruleStack: result.ruleStack,\n    resolvedTokens,\n  };\n}\n\nexport function renderLineTokens(\n  tokens: Array<HighlightedToken>,\n  themeType: 'light' | 'dark'\n): (HTMLElement | string)[] {\n  return tokens.map(([char, fg, textContent]) => {\n    if (char === 0 && fg === '') {\n      if (textContent === '') {\n        return h('br');\n      }\n      return textContent;\n    }\n    return h('span', {\n      dataset: {\n        char: char.toString(),\n      },\n      style: `--diffs-token-${themeType}:${fg};`,\n      textContent: textContent,\n    });\n  });\n}\n\n// Shiki special-cases `text` and `ansi` in codeToHast but does not expose grammars.\nfunction isGrammarlessLanguage(languageId: string): boolean {\n  return languageId === 'text' || languageId === 'ansi';\n}\n"],"mappings":";;;;;;AA8BA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,sBAAsB;CAE7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,cAA4B,CAAC,QAAQ;CACrC,YAAoB;CACpB,aAAsB;CACtB,YAAqB;CACrB,mBAA2B;CAC3B;CACA,+BAAuC;CACvC,6BAAsC;CAEtC,sBAAsB,SAAS,OAAO,gBAA8B;EAClE,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,UAAU,KAAK,IACnB,eAAe,WAAW,WAAW,eAAe,YACpD,MAAKA,aAAc,UACpB;AACD,MACE,MAAKC,YAAa,UAClB,CAAC,sBAAsB,MAAKD,aAAc,WAAW,EACrD;AACA,SAAM,MAAKE,YAAa,aAAa,MAAKF,aAAc,WAAW;AACnE,SAAKC,UAAW,MAAKC,YAAa,YAChC,MAAKF,aAAc,WACpB;;AAEH,QAAKG,gBAAiB,QAAQ;IAC7B,IAAI;CAEP,cAAc,EAAE,WAAkC;AAChD,MAAI,OAAO,SAAS,YAAY,SAAS,KACvC;EAEF,MAAM,EAAE,MAAM,UAAU;AAIxB,MACE,SAAS,cACT,OAAO,UAAU,YACjB,UAAU,MAAKC,gBAEf,OAAKC,mBAAoB,MAAM;;CAInC,IAAI,YAA8B;AAChC,SAAO,MAAKC;;CAGd,YAAY,EACV,aACA,aACA,cACA,UACA,iBACA,WACuB;EACvB,MAAM,EACJ,YAAY,UACZ,QAAQ,gBACR,wBAAwB,QACtB;AACJ,QAAKC,iBAAkB,OAAO,WAAW,+BAA+B;AACxE,MAAI,cAAc,SAChB,OAAKD,YAAa,MAAKC,eAAgB,UAAU,SAAS;MAE1D,OAAKD,YAAa;AAEpB,MAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,WAAW,IAAI,kBAAkB,cAAc;AACnD,SAAK,MAAM,EAAE,MAAM,mBAAmB,UACpC,KACE,SAAS,gBACT,kBAAkB,SACjB,kBAAkB,WAAW,cAAc,WAAW,QAAQ,GAC/D;KACA,MAAME,cACJ,iBAAiB,SAAS,KAAK,CAAC,gBAAgB,SAC5C,SACA;AACN,WAAKC,gBAAiB,MAAMD,cAAYA,YAAU;AAClD;;KAGJ;AACF,YAAS,QAAQ,SAAS,iBAAiB,EAAE,YAAY,MAAM,CAAC;AAChE,YAAS,QAAQ,SAAS,MAAM,EAAE,YAAY,MAAM,CAAC;AACrD,SAAKE,WAAY,CACf,iBAAiB,MAAKH,gBAAiB,WAAW,MAAM;IACtD,MAAMC,cAAY,EAAE,UAAU,SAAS;AACvC,UAAKC,gBAAiB,MAAMD,cAAYA,YAAU;KAClD,QACI,SAAS,YAAY,CAC5B;;AAEH,QAAKN,cAAe;AACpB,QAAKF,eAAgB;AACrB,QAAKW,wBAAyB;AAC9B,QAAKC,WAAY;AACjB,QAAKC,kBAAmB;AACxB,QAAKC,QAAS,WAAW;AACzB,MACE,CAAC,sBAAsB,aAAa,WAAW,IAC/C,YAAY,oBAAoB,CAAC,SAAS,aAAa,WAAW,CAElE,OAAKb,UAAW,YAAY,YAAY,aAAa,WAAW;AAElE,QAAKc,WAAY,EAAE;AACnB,QAAKC,SAAU,OAAO,UAAU,WAAW,QAAQ,MAAM,MAAKV,WAAY;;CAK5E,iBAAiB,WAAmB,WAA6B;AAC/D,QAAKA,YAAa;AAClB,QAAKU,SAAU,UAAU;AACzB,OAAK,wBAAwB;AAC7B,QAAKC,aAAc,CAAC,QAAQ;AAC5B,MAAI,MAAKhB,YAAa,UAAa,MAAKD,aAAc,YAAY,EAChE,OAAKkB,2BAA4B,EAAE;;CAIvC,UAAU,WAAmB;AAC3B,QAAKH,WAAY,MAAKb,YAAa,SAAS,UAAU,CAAC;EACvD,MAAM,EAAE,SAAS,EAAE,KAAK,MAAKA,YAAa,SAAS,UAAU;EAC7D,MAAM,sBAAsB,OAAO;EACnC,MAAM,0BAA0B,OAAO;EACvC,MAAM,mBAAmB,OAAO;EAChC,MAAM,yBAAyB,OAAO;EACtC,MAAM,mBAAmB,OAAO;EAChC,MAAM,sBAAsB,OAAO;EACnC,MAAM,+BACJ,OAAO;EACT,MAAM,iBAAiB,OAAO;EAC9B,MAAM,iBAAiB,OAAO;EAC9B,MAAM,oBAAoB,OAAO;EACjC,MAAM,kBAAkB,OAAO;AAC/B,QAAKU,SAAU;qCACkB,uBAAuB,uBAAuB;0CACzC,2BAA2B,uBAAuB;uCACrD,oBAAoB,yBAAyB;8CACtC,2BAA2B,wCAAwC;8CACnE,0BAA0B,mCAAmC;iCAC1E,uBAAuB,QAAQ;2CACrB,gCAAgC,QAAQ;kCACjD,oBAAoB,QAAQ;gCAC9B,kBAAkB,QAAQ;gCAC1B,kBAAkB,QAAQ;mCACvB,qBAAqB,QAAQ;iCAC/B,mBAAmB,QAAQ;OACrD;;CAGL,UAAgB;AACd,OAAK,wBAAwB;AAC7B,QAAKO,uBAAwB;AAC7B,QAAKT,UAAW,SAAS,YAAY,SAAS,CAAC;AAC/C,QAAKA,WAAY;;CAKnB,SACE,QACA,aACsC;AACtC,MACE,MAAKT,YAAa,UAClB,CAAC,sBAAsB,MAAKD,aAAc,WAAW,CAErD,OAAM,IAAI,MAAM,qBAAqB;EAGvC,MAAM,EAAE,cAAc,MAAKA;EAC3B,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,qBACJ,eAAe,WACX,YACA,KAAK,IAAI,eAAe,YAAY,UAAU;EAEpD,MAAM,aAAa,OAAO;EAC1B,MAAM,YAAY,KAAK,IAAI,cAAc,WAAW;EACpD,MAAM,wBACJ,gBAAgB,UAChB,eAAe,YACf,OAAO,YAAY,KACnB,aAAa,sBACb,OAAO,WAAW;EACpB,MAAM,uBAAuB,OAAO,cAAc;EAClD,MAAM,0BACJ,wBACA,gBAAgB,UAChB,cAAc;EAChB,MAAMoB,oBAAiD,uBAClD,OAAO,qBAAqB,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC,GAC3D,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC;EAClC,IAAI,mBAAmB;AACvB,MAAI,aAAa,WACf;QAAK,MAAM,CAAC,YAAY,aAAa,kBACnC,KAAI,aAAa,UACf,oBAAmB,KAAK,IACtB,kBACA,KAAK,IAAI,UAAU,YAAY,EAAE,CAClC;;EAIP,MAAM,4BACJ,oBAAoB,eACnB,wBAAwB,OAAO,YAAY;AAC9C,MAAI,qBACF,OAAKjB,gBAAiB,WAAW;OAC5B;AACL,SAAKc,WAAY,SAAS,KAAK,IAC7B,MAAKA,WAAY,QACjB,aAAa,EACd;AACD,OAAI,gBAAgB,UAAa,cAAc,UAC7C,OAAKd,gBAAiB,UAAU;;EAIpC,IAAI,oBAAoB;EACxB,IAAI,yBAAyB,kBAAkB,mBAAmB;EAClE,IAAIkB;EACJ,IAAI,8BAA8B;EAClC,IAAI,OAAO,uBACP,kBAAkB,mBAAmB,KACrC;EACJ,IAAI,QAAQ,MAAKJ,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,MAAMK,6BAAmD,IAAI,KAAK;EAClE,MAAMC,sBAEU,4CAA4B,IAAI,KAAK,GAAG;AACxD,MAAI,wBAAwB,UAAa,CAAC,sBAAsB;GAC9D,MAAM,eAAe,KAAK,IACxB,mBAAmB,GACnB,WACA,mBACD;AACD,OAAI,eAAe,YAAY;AAC7B,UAAKpB,gBAAiB,aAAa;IACnC,IAAI,gBAAgB;IACpB,IAAI,iBAAiB,MAAKc,WAAY,kBAAkB;AACxD,WAAO,gBAAgB,cAAc,iBAAiB;KACpD,MAAM,WAAW,MAAKO,eAAgB,eAAe,eAAe;AACpE,sBAAiB,SAAS;AAC1B,yBAAoB,IAAI,eAAe,SAAS,eAAe;;AAEjE,QAAI,wBACF,OAAKP,WAAY,gBAAgB;;;AAIvC,SAAO,OAAO,qBAAsB;GAClC,MAAM,oBAAoB,uBACtB,MAAKA,WAAY,OAAO,KACxB;AACJ,OAAI,wBACF,OAAKA,WAAY,QAAQ;GAG3B,MAAM,EAAE,gBAAgB,OAAO,cAAc,MAAKO,eAChD,MACA,MACD;AACD,WAAQ;AAER,OAAI,QAAQ,UACV,YAAW,IAAI,MAAM,eAAe;OAEpC,sBAAqB,IAAI,MAAM,eAAe;AAGhD,OAAI,wBACF,OAAKP,WAAY,OAAO,KAAK;AAE/B,aACE,QAAQ,0BACR,wBACA,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC,OAAI,SAAS;AACX;IACA,MAAM,YAAY,kBAAkB;AACpC,QAAI,cAAc,OAChB;AAEF,QAAI,UAAU,MAAM,oBAAoB;AACtC,2BAAsB,UAAU;AAChC,mCAA8B;AAC9B;;AAEF,QAAI,MAAKA,WAAY,UAAU,QAAQ,QAAW;AAChD,8BAAyB,UAAU;AACnC;WACK;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,8BAAyB,UAAU;;AAErC,cAAU;AACV;;AAEF;;AAGF,MAAI,wBACF,KAAI,OAAO,mBACT,OAAKA,WAAY,OAAO,KAAK;MAE7B,OAAKA,WAAY,QAAQ;AAI7B,MAAI,wBAAwB,UAAa,oBAAoB,OAAO,EAClE,OAAKJ,gBAAiB,qBAAqB,MAAKP,UAAW;AAG7D,MAAI,wBAAwB,OAC1B,OAAKY,2BACH,qBACA,mBACA,4BACD;WACQ,CAAC,WAAW,OAAO,WAAW;GACvC,MAAM,iBACJ,yBAAyB,cAAc,YACnC,qBACA,aAAa,aAAa,CAAC,uBACzB,aACA;AACR,SAAKA,2BACH,gBACA,uBAAuB,oBAAoB,QAC3C,kBACD;;AAGH,SAAO;;CAGT,mBAAmB,aAAiC;AAClD,QAAKO,mBAAoB,YAAY;;CAGvC,yBAA+B;AAC7B,MAAI,MAAKC,UACP;AAEF,QAAKA,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKX,uBAAwB;;CAG/B,0BAAgC;AAC9B,MAAI,MAAKO,aAAc,MAAKC,SAC1B;AAEF,MAAI,MAAKb,MACP,SAAQ,IAAI,iDAAiD,EAC3D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;;CAGnB,2BAAiC;AAC/B,MACE,MAAKD,aACL,CAAC,MAAKC,YACN,MAAK1B,YAAa,UAClB,MAAK2B,WAAY,EAEjB;AAEF,MAAI,MAAKd,MACP,SAAQ,IAAI,kDAAkD,EAC5D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;AACjB,QAAKI,oBAAqB,MAAK3B,gBAAiB;;CAGlD,yBAA+B;AAC7B,MAAI,MAAK4B,0BACP;AAEF,aAAW,iBAAiB,WAAW,MAAKC,UAAW;AACvD,QAAKD,4BAA6B;;CAGpC,yBAA+B;AAC7B,MAAI,CAAC,MAAKA,0BACR;AAEF,aAAW,oBAAoB,WAAW,MAAKC,UAAW;AAC1D,QAAKD,4BAA6B;;CAGpC,qBAAqB,OAAqB;AAExC,aAAW,YAAY;GAAE,MAAM;GAAY;GAAO,CAAC;;CAGrD,4BACE,WACA,mBACA,oBAAoB,GACd;AACN,MAAI,sBAAsB,MAAKhC,aAAc,WAAW,CACtD;EAGF,MAAM,QAAQ,EAAE,MAAKI;AAErB,MAAI,MAAKU,MACP,SAAQ,IAAI,oDAAoD;GAC9D;GACA;GACA;GACA;GACD,CAAC;AAGJ,QAAKY,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKI,uBAAwB;AAC7B,QAAKH,oBAAqB,MAAM;;CAGlC,gBACE,MACA,OACgE;EAChE,MAAM,WAAW,MAAK/B,aAAc,YAAY,KAAK;AACrD,MAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,WAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC;IAAE;IAAO;;AAEvD,MACE,MAAKV,YAAa,UAClB,aAAa,MACb,SAAS,MAAM,KAAK,GAEpB,QAAO;GAAE,gBAAgB,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC;GAAE;GAAO;EAEvD,MAAM,SAAS,aACb,MAAKA,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,SAAO;GACL,gBAAgB,OAAO;GACvB,OAAO,OAAO;GACf;;CAGH,iBAAiB,OAAe;EAC9B,MAAM,eAAe,KAAK,IACxB,KAAK,IAAI,GAAG,MAAM,EAClB,MAAKf,aAAc,UACpB;AACD,MAAI,MAAKiB,WAAY,SAAS,gBAAgB,MAAKhB,YAAa,OAC9D;EAEF,IAAI,OAAO,MAAKgB,WAAY,SAAS;EACrC,IAAI,QAAQ,MAAKA,WAAY,SAAS;AACtC,SAAO,OAAO,cAAc,QAAQ;AAClC,SAAKA,WAAY,QAAQ;GACzB,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OACE,SAAS,UAAU,MAAKW,yBACxB,aAAa,MACb,SAAS,MAAM,KAAK,GAEpB,SAAQ,MAAKV,QAAS,cACpB,UACA,OACA,gBAAgB,oBACjB,CAAC;;AAGN,QAAKgB,WAAY,QAAQ;;CAG3B,oBAAoB,OAAe;AACjC,MACE,MAAKS,aACL,MAAKC,YACL,MAAK1B,YAAa,UAClB,UAAU,MAAKG,gBAEf;EAGF,MAAM,IAAI,YAAY,KAAK;EAC3B,MAAM,wBAAQ,IAAI,KAAsC;EACxD,MAAM,aAAa,MAAKJ,aAAc;EACtC,MAAM,oBAAoB,MAAK6B;EAE/B,IAAI,OAAO,MAAKD;EAChB,IAAI,QAAQ,MAAKX,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,IAAI,oBAAoB,MAAKa;EAC7B,IAAI,yBAAyB,oBAAoB,qBAAqB;AACtE,SAAO,OAAO,aAAc;AAC1B,SAAKb,WAAY,QAAQ;GAEzB,MAAM,oBACJ,2BAA2B,SACvB,MAAKA,WAAY,OAAO,KACxB;GACN,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,YAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAM,IAAI,MAAM,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC,CAAC;cAC3B,aAAa,MAAM,SAAS,MAAM,KAAK,GAChD,OAAM,IAAI,MAAM,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC,CAAC;QAC/B;IACL,MAAM,MAAM,aACV,MAAKV,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,UAAM,IAAI,MAAM,IAAI,eAAe;AACnC,YAAQ,IAAI;;AAGd,SAAKE,WAAY,OAAO,KAAK;AAC7B,aACE,2BAA2B,UAC3B,QAAQ,0BACR,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC;AACA,OAAI,SAAS;AACX;IACA,MAAM,YAAY,oBAAoB;AACtC,QAAI,cAAc,OAChB;AAEF,6BAAyB,UAAU;AACnC,QAAI,MAAKA,WAAY,UAAU,QAAQ,OACrC,WAAU;SACL;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,eAAU;AACV;;;AAKJ,OAAI,YAAY,KAAK,GAAG,IAAI,EAC1B;;AAIJ,QAAKJ,gBAAiB,OAAO,MAAKP,UAAW;AAC7C,MAAI,MAAKoB,aAAc,MAAKC,YAAa,UAAU,MAAKvB,gBACtD;AAGF,MAAI,WAAW,QAAQ,YAAY;AACjC,QAAK,wBAAwB;AAC7B;;AAGF,QAAKwB,WAAY;AACjB,QAAKE,8BAA+B;AACpC,QAAKC,oBAAqB,MAAM;;;AAIpC,SAAgB,aACd,SACA,UACA,UACA,YACA,WAIA;CACA,MAAM,SAAS,QAAQ,cAAc,UAAU,YAAY,UAAU;AACrE,KAAI,OAAO,aACT,SAAQ,KACN,oDAAoD,SAAS,UAAU,GAAG,IAAI,GAC/E;CAEH,MAAM,YAAY,OAAO;CACzB,MAAM,eAAe,UAAU,SAAS;CACxC,MAAMI,iBAA0C,EAAE;AAClD,MAAK,IAAI,IAAI,GAAG,IAAI,cAAc,KAAK;EACrC,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,aACJ,IAAI,IAAI,eAAe,UAAU,IAAI,IAAI,KAAK,SAAS;AACzD,MAAI,WAAW,WAEb;EAEF,MAAM,WAAW,UAAU,IAAI,IAAI;EAEnC,MAAM,KAAK,SADA,qBAAqB,cAAc,SAAS;EAEvD,MAAM,YAAY,SAAS,MAAM,QAAQ,WAAW;AACpD,iBAAe,KAAK;GAAC;GAAQ;GAAI;GAAU,CAAC;;AAE9C,QAAO;EACL,WAAW,OAAO;EAClB;EACD;;AAGH,SAAgB,iBACd,QACA,WAC0B;AAC1B,QAAO,OAAO,KAAK,CAAC,MAAM,IAAI,iBAAiB;AAC7C,MAAI,SAAS,KAAK,OAAO,IAAI;AAC3B,OAAI,gBAAgB,GAClB,QAAO,EAAE,KAAK;AAEhB,UAAO;;AAET,SAAO,EAAE,QAAQ;GACf,SAAS,EACP,MAAM,KAAK,UAAU,EACtB;GACD,OAAO,iBAAiB,UAAU,GAAG,GAAG;GAC3B;GACd,CAAC;GACF;;AAIJ,SAAS,sBAAsB,YAA6B;AAC1D,QAAO,eAAe,UAAU,eAAe"}
+{"version":3,"file":"tokenzier.js","names":["#textDocument","#grammar","#highlighter","#buildStateStack","#backgroundJobId","#backgroundTokenize","#themeType","#mediaQueryList","#emitThemeChange","#disposes","#tokenizeMaxLineLength","#setStyle","#onDeferTokenize","#onThemeChange","#debug","#ensureGrammar","#colorMap","#setTheme","#stateStack","#scheduleBackgroundTokenize","#themeName","#detachMessageListener","#tokenizeLineAt","#prebuildStateStack","#isStopped","#isPaused","#lastLine","#backgroundChangedLineRanges","#backgroundChangedRangeIndex","#postTokenizeMessage","#isMessageListenerAttached","#onMessage","#attachMessageListener"],"sources":["../../src/editor/tokenzier.ts"],"sourcesContent":["import {\n  EncodedTokenMetadata,\n  type IGrammar,\n  INITIAL,\n  type StateStack,\n} from 'shiki/textmate';\n\nimport { DEFAULT_THEMES } from '../constants';\nimport type {\n  BaseCodeOptions,\n  DiffsHighlighter,\n  HighlightedToken,\n  RenderRange,\n} from '../types';\nimport type { TextDocument, TextDocumentChange } from './textDocument';\nimport { addEventListener, debounce, h } from './utils';\n\nexport interface EditorTokenizerProps {\n  highlighter: DiffsHighlighter;\n  textDocument: TextDocument<unknown>;\n  codeOptions: BaseCodeOptions;\n  setStyle: (style: string) => void;\n  onDeferTokenize: (\n    lines: Map<number, Array<HighlightedToken>>,\n    themeType: 'dark' | 'light'\n  ) => void;\n  // Fired after the active theme (light/dark mode or theme name) changes and the\n  // new theme CSS has been applied. Lets the editor recompute overlay pieces\n  // that captured a resolved theme color, e.g. rounded selection corner masks.\n  onThemeChange?: () => void;\n  __debug?: boolean;\n}\n\n/** Stoppable code tokenizer for the editor */\nexport class EditorTokenizer {\n  static TOKENIZE_TIME_LIMIT = 500;\n\n  #highlighter: DiffsHighlighter;\n  #grammar: IGrammar | undefined;\n  #mediaQueryList: MediaQueryList;\n  #themeType: 'light' | 'dark';\n  // The resolved name of the theme currently applied to the editor (e.g.\n  // `github-light`). Tracked so `syncTheme` can detect a host-driven theme swap\n  // even when the light/dark mode itself is unchanged.\n  #themeName = '';\n  #colorMap: string[];\n  #textDocument: TextDocument<unknown>;\n  #tokenizeMaxLineLength: number;\n  #setStyle: EditorTokenizerProps['setStyle'];\n  #onDeferTokenize: EditorTokenizerProps['onDeferTokenize'];\n  #onThemeChange: EditorTokenizerProps['onThemeChange'];\n  #debug: boolean;\n  #disposes?: (() => void)[];\n\n  // state\n  #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index\n  #lastLine: number = -1;\n  #isStopped: boolean = true;\n  #isPaused: boolean = false;\n  #backgroundJobId: number = 0;\n  #backgroundChangedLineRanges: readonly [number, number][] | undefined;\n  #backgroundChangedRangeIndex: number = 0;\n  #isMessageListenerAttached: boolean = false;\n\n  #prebuildStateStack = debounce(async (renderRange?: RenderRange) => {\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const endLine = Math.min(\n      totalLines === Infinity ? Infinity : startingLine + totalLines,\n      this.#textDocument.lineCount\n    );\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      await this.#highlighter.loadLanguage(this.#textDocument.languageId);\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n    this.#buildStateStack(endLine);\n  }, 500);\n\n  #onMessage = ({ data }: MessageEvent<unknown>) => {\n    if (typeof data !== 'object' || data === null) {\n      return;\n    }\n    const { type, jobId } = data as {\n      type?: unknown;\n      jobId?: unknown;\n    };\n    if (\n      type === 'tokenize' &&\n      typeof jobId === 'number' &&\n      jobId === this.#backgroundJobId\n    ) {\n      this.#backgroundTokenize(jobId);\n    }\n  };\n\n  get themeType(): 'light' | 'dark' {\n    return this.#themeType;\n  }\n\n  constructor({\n    codeOptions,\n    highlighter,\n    textDocument,\n    setStyle,\n    onDeferTokenize,\n    onThemeChange,\n    __debug,\n  }: EditorTokenizerProps) {\n    const {\n      themeType = 'system',\n      theme = DEFAULT_THEMES,\n      tokenizeMaxLineLength = 1000,\n    } = codeOptions;\n    this.#mediaQueryList = window.matchMedia('(prefers-color-scheme: dark)');\n    if (themeType === 'system') {\n      this.#themeType = this.#mediaQueryList.matches ? 'dark' : 'light';\n    } else {\n      this.#themeType = themeType;\n    }\n    // Only track the document/system color scheme when the surface follows it\n    // (`themeType: 'system'`). A surface pinned to an explicit 'dark'/'light'\n    // theme keeps that theme regardless of the page, so re-tokenizing after an\n    // edit must emit the same `--diffs-token-{theme}` variable the SSR markup\n    // used; otherwise the edited tokens fall back to the default foreground.\n    if (typeof theme !== 'string' && themeType === 'system') {\n      const observer = new MutationObserver((mutations) => {\n        for (const { type, attributeName } of mutations) {\n          if (\n            type === 'attributes' &&\n            attributeName !== null &&\n            (attributeName === 'class' || attributeName.startsWith('data-'))\n          ) {\n            const themeType =\n              getComputedStyle(document.body).colorScheme === 'dark'\n                ? 'dark'\n                : 'light';\n            this.#emitThemeChange(theme[themeType], themeType);\n            break;\n          }\n        }\n      });\n      observer.observe(document.documentElement, { attributes: true });\n      observer.observe(document.body, { attributes: true });\n      this.#disposes = [\n        addEventListener(this.#mediaQueryList, 'change', (e) => {\n          const themeType = e.matches ? 'dark' : 'light';\n          this.#emitThemeChange(theme[themeType], themeType);\n        }),\n        () => observer.disconnect(),\n      ];\n    }\n    this.#highlighter = highlighter;\n    this.#textDocument = textDocument;\n    this.#tokenizeMaxLineLength = tokenizeMaxLineLength;\n    this.#setStyle = setStyle;\n    this.#onDeferTokenize = onDeferTokenize;\n    this.#onThemeChange = onThemeChange;\n    this.#debug = __debug ?? false;\n    this.#ensureGrammar();\n    this.#colorMap = [];\n    this.#setTheme(typeof theme === 'string' ? theme : theme[this.#themeType]);\n  }\n\n  // By default, diffs components support dual themes, but the tokenizer only renders\n  // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document.\n  #emitThemeChange(themeName: string, themeType: 'light' | 'dark') {\n    this.#themeType = themeType;\n    this.#setTheme(themeName);\n    this.stopBackgroundTokenize();\n    this.#stateStack = [INITIAL];\n    if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) {\n      this.#scheduleBackgroundTokenize(0);\n    }\n    // The theme CSS is now applied, so overlay pieces that captured a resolved\n    // theme color (e.g. rounded selection corner masks) can recompute against\n    // the new colors instead of keeping the old light/dark value.\n    this.#onThemeChange?.();\n  }\n\n  // Re-apply the editor's theme from the surface's current code options. Edit\n  // mode reuses a single tokenizer across re-renders, so when the host swaps the\n  // theme — a theme picker, a light/dark toggle, etc. — we must recompute the\n  // active theme and re-tokenize. Without this the editor keeps rendering the\n  // theme it captured when it first attached (stale line-highlight background\n  // and token colors). System-driven changes are still handled by the\n  // observers wired up in the constructor; this covers explicit `themeType`/\n  // `theme` option changes that those observers don't see.\n  syncTheme(codeOptions: BaseCodeOptions): void {\n    const { themeType = 'system', theme = DEFAULT_THEMES } = codeOptions;\n    const nextThemeType =\n      themeType === 'system'\n        ? this.#mediaQueryList.matches\n          ? 'dark'\n          : 'light'\n        : themeType;\n    const nextThemeName =\n      typeof theme === 'string' ? theme : theme[nextThemeType];\n    if (\n      nextThemeType === this.#themeType &&\n      nextThemeName === this.#themeName\n    ) {\n      return;\n    }\n    this.#emitThemeChange(nextThemeName, nextThemeType);\n  }\n\n  #setTheme(themeName: string) {\n    this.#themeName = themeName;\n    this.#colorMap = this.#highlighter.setTheme(themeName).colorMap;\n    const { colors = {} } = this.#highlighter.getTheme(themeName);\n    const selectionBackground = colors['editor.selectionBackground'];\n    const lineHighlightBackground = colors['editor.lineHighlightBackground'];\n    const cursorForeground = colors['editorCursor.foreground'];\n    const findMatchBackground = colors['editor.findMatchBackground'];\n    const findMatchHighlightBackground =\n      colors['editor.findMatchHighlightBackground'];\n    const hintForeground = colors['editorHint.foreground'];\n    const infoForeground = colors['editorInfo.foreground'];\n    const warningForeground = colors['editorWarning.foreground'];\n    const errorForeground = colors['editorError.foreground'];\n    this.#setStyle(`:host {\n      --diffs-editor-selection-bg: ${selectionBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-highlight-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-match-bg: ${findMatchBackground ?? 'unset'};\n      --diffs-editor-match-highlight-bg: ${findMatchHighlightBackground ?? 'unset'};\n      --diffs-editor-cursor-fg: ${cursorForeground ?? 'unset'};\n      --diffs-editor-hint-fg: ${hintForeground ?? 'unset'};\n      --diffs-editor-info-fg: ${infoForeground ?? 'unset'};\n      --diffs-editor-warning-fg: ${warningForeground ?? 'unset'};\n      --diffs-editor-error-fg: ${errorForeground ?? 'unset'};\n    }`);\n  }\n\n  cleanUp(): void {\n    this.stopBackgroundTokenize();\n    this.#detachMessageListener();\n    this.#disposes?.forEach((dispose) => dispose());\n    this.#disposes = undefined;\n  }\n\n  // to use `tokenize`, call `prebuildStateStackMap` first to prebuild\n  // the state stack map for the given render range.\n  tokenize(\n    change: TextDocumentChange,\n    renderRange?: RenderRange\n  ): Map<number, Array<HighlightedToken>> {\n    this.#ensureGrammar();\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      throw new Error(\n        `Grammar for language \"${this.#textDocument.languageId}\" not loaded`\n      );\n    }\n\n    const { lineCount } = this.#textDocument;\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const renderRangeEndLine =\n      totalLines === Infinity\n        ? lineCount\n        : Math.min(startingLine + totalLines, lineCount);\n\n    const dirtyStart = change.startLine;\n    const viewStart = Math.max(startingLine, dirtyStart);\n    const crossesRenderRangeEnd =\n      renderRange !== undefined &&\n      totalLines !== Infinity &&\n      change.lineDelta > 0 &&\n      dirtyStart < renderRangeEndLine &&\n      change.endLine >= renderRangeEndLine;\n    const canReuseCachedStates = change.lineDelta === 0;\n    const canCacheTokenizedStates =\n      canReuseCachedStates ||\n      renderRange === undefined ||\n      dirtyStart >= viewStart;\n    const changedLineRanges: readonly [number, number][] = canReuseCachedStates\n      ? (change.changedLineRanges ?? [[dirtyStart, change.endLine]])\n      : [[dirtyStart, change.endLine]];\n    let offscreenSyncEnd = -1;\n    if (dirtyStart < viewStart) {\n      for (const [rangeStart, rangeEnd] of changedLineRanges) {\n        if (rangeStart < viewStart) {\n          offscreenSyncEnd = Math.max(\n            offscreenSyncEnd,\n            Math.min(rangeEnd, viewStart - 1)\n          );\n        }\n      }\n    }\n    const shouldFlushOffscreenLines =\n      offscreenSyncEnd >= dirtyStart &&\n      (canReuseCachedStates || change.lineDelta < 0);\n    if (canReuseCachedStates) {\n      this.#buildStateStack(dirtyStart);\n    } else {\n      this.#stateStack.length = Math.min(\n        this.#stateStack.length,\n        dirtyStart + 1\n      );\n      if (renderRange === undefined || dirtyStart >= viewStart) {\n        this.#buildStateStack(viewStart);\n      }\n    }\n\n    let changedRangeIndex = 0;\n    let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];\n    let backgroundStartLine: number | undefined;\n    let backgroundChangedRangeIndex = 0;\n    let line = canReuseCachedStates\n      ? changedLineRanges[changedRangeIndex][0]\n      : viewStart;\n    let settled = false;\n    const dirtyLines: Map<number, Array<HighlightedToken>> = new Map();\n    const offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined = shouldFlushOffscreenLines ? new Map() : undefined;\n    if (offscreenDirtyLines !== undefined && !canReuseCachedStates) {\n      const offscreenEnd = Math.min(\n        offscreenSyncEnd + 1,\n        viewStart,\n        renderRangeEndLine\n      );\n      if (offscreenEnd > dirtyStart) {\n        this.#buildStateStack(offscreenEnd);\n        let offscreenLine = dirtyStart;\n        let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;\n        for (; offscreenLine < offscreenEnd; offscreenLine++) {\n          const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);\n          offscreenState = resolved.state;\n          offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);\n        }\n        if (canCacheTokenizedStates) {\n          this.#stateStack[offscreenEnd] = offscreenState;\n        }\n      }\n    }\n    // Seed the loop's grammar state after the offscreen flush, not before it.\n    // When a delete's removed lines reach the viewport's first line, the flush\n    // rebuilds the cached state up to `line`; reading it earlier would capture\n    // the truncated INITIAL state and color the viewport as if outside an open\n    // construct (block comment, template literal) it is actually inside.\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < renderRangeEndLine; ) {\n      const previousNextState = canReuseCachedStates\n        ? this.#stateStack[line + 1]\n        : undefined;\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line] = state;\n      }\n\n      const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(\n        line,\n        state\n      );\n      state = nextState;\n\n      if (line >= viewStart) {\n        dirtyLines.set(line, resolvedTokens);\n      } else {\n        offscreenDirtyLines?.set(line, resolvedTokens);\n      }\n\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line + 1] = state;\n      }\n      settled =\n        line >= currentChangedRangeEnd &&\n        canReuseCachedStates &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        if (nextRange[0] >= renderRangeEndLine) {\n          backgroundStartLine = nextRange[0];\n          backgroundChangedRangeIndex = changedRangeIndex;\n          break;\n        }\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          currentChangedRangeEnd = nextRange[1];\n          line++;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          currentChangedRangeEnd = nextRange[1];\n        }\n        settled = false;\n        continue;\n      }\n      line++;\n    }\n\n    if (canCacheTokenizedStates) {\n      if (line < renderRangeEndLine) {\n        this.#stateStack[line + 1] = state;\n      } else {\n        this.#stateStack[line] = state;\n      }\n    }\n\n    if (offscreenDirtyLines !== undefined && offscreenDirtyLines.size > 0) {\n      this.#onDeferTokenize(offscreenDirtyLines, this.#themeType);\n    }\n\n    if (backgroundStartLine !== undefined) {\n      this.#scheduleBackgroundTokenize(\n        backgroundStartLine,\n        changedLineRanges,\n        backgroundChangedRangeIndex\n      );\n    } else if (!settled && line < lineCount) {\n      const backgroundLine =\n        crossesRenderRangeEnd && dirtyStart >= viewStart\n          ? renderRangeEndLine\n          : dirtyStart < viewStart && !canReuseCachedStates\n            ? dirtyStart\n            : line;\n      this.#scheduleBackgroundTokenize(\n        backgroundLine,\n        canReuseCachedStates ? changedLineRanges : undefined,\n        changedRangeIndex\n      );\n    }\n\n    return dirtyLines;\n  }\n\n  prebuildStateStack(renderRange?: RenderRange): void {\n    this.#ensureGrammar();\n    this.#prebuildStateStack(renderRange);\n  }\n\n  #ensureGrammar(): void {\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId) &&\n      this.#highlighter\n        .getLoadedLanguages()\n        .includes(this.#textDocument.languageId)\n    ) {\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n  }\n\n  stopBackgroundTokenize(): void {\n    if (this.#isStopped) {\n      return;\n    }\n    this.#isStopped = true;\n    this.#isPaused = false;\n    this.#lastLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#detachMessageListener();\n  }\n\n  pauseBackgroundTokenize(): void {\n    if (this.#isStopped || this.#isPaused) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization paused', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = true;\n  }\n\n  resumeBackgroundTokenize(): void {\n    if (\n      this.#isStopped ||\n      !this.#isPaused ||\n      this.#grammar === undefined ||\n      this.#lastLine < 0\n    ) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization resumed', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = false;\n    this.#postTokenizeMessage(this.#backgroundJobId);\n  }\n\n  #attachMessageListener(): void {\n    if (this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.addEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = true;\n  }\n\n  #detachMessageListener(): void {\n    if (!this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.removeEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = false;\n  }\n\n  #postTokenizeMessage(jobId: number): void {\n    // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay\n    globalThis.postMessage({ type: 'tokenize', jobId });\n  }\n\n  #scheduleBackgroundTokenize(\n    startLine: number,\n    changedLineRanges?: readonly [number, number][],\n    changedRangeIndex = 0\n  ): void {\n    if (isGrammarlessLanguage(this.#textDocument.languageId)) {\n      return;\n    }\n\n    const jobId = ++this.#backgroundJobId;\n\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization scheduled', {\n        jobId,\n        startLine,\n        changedLineRanges,\n        changedRangeIndex,\n      });\n    }\n\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = startLine;\n    this.#backgroundChangedLineRanges = changedLineRanges;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #tokenizeLineAt(\n    line: number,\n    state: StateStack\n  ): { resolvedTokens: Array<HighlightedToken>; state: StateStack } {\n    const lineText = this.#textDocument.getLineText(line);\n    if (lineText.length > this.#tokenizeMaxLineLength) {\n      console.warn(\n        `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n      );\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    if (\n      this.#grammar === undefined ||\n      lineText === '' ||\n      lineText.trim() === ''\n    ) {\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    const result = tokenizeLine(\n      this.#grammar,\n      this.#colorMap,\n      lineText,\n      state,\n      EditorTokenizer.TOKENIZE_TIME_LIMIT\n    );\n    return {\n      resolvedTokens: result.resolvedTokens,\n      state: result.ruleStack,\n    };\n  }\n\n  #buildStateStack(endAt: number) {\n    const boundedEndAt = Math.min(\n      Math.max(0, endAt),\n      this.#textDocument.lineCount\n    );\n    if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) {\n      return;\n    }\n    let line = this.#stateStack.length - 1;\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < boundedEndAt; line++) {\n      this.#stateStack[line] = state;\n      const lineText = this.#textDocument.getLineText(line);\n      if (\n        lineText.length <= this.#tokenizeMaxLineLength &&\n        lineText !== '' &&\n        lineText.trim() !== ''\n      ) {\n        state = this.#grammar.tokenizeLine2(\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        ).ruleStack;\n      }\n    }\n    this.#stateStack[line] = state;\n  }\n\n  #backgroundTokenize(jobId: number) {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar === undefined ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    const t = performance.now();\n    const lines = new Map<number, Array<HighlightedToken>>();\n    const totalLines = this.#textDocument.lineCount;\n    const changedLineRanges = this.#backgroundChangedLineRanges;\n\n    let line = this.#lastLine;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    let changedRangeIndex = this.#backgroundChangedRangeIndex;\n    let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];\n    for (; line < totalLines; ) {\n      this.#stateStack[line] = state;\n\n      const previousNextState =\n        currentChangedRangeEnd !== undefined\n          ? this.#stateStack[line + 1]\n          : undefined;\n      const lineText = this.#textDocument.getLineText(line);\n      if (lineText.length > this.#tokenizeMaxLineLength) {\n        console.warn(\n          `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n        );\n        lines.set(line, [[0, '', lineText]]);\n      } else if (lineText === '' || lineText.trim() === '') {\n        lines.set(line, [[0, '', lineText]]);\n      } else {\n        const ret = tokenizeLine(\n          this.#grammar,\n          this.#colorMap,\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        );\n        lines.set(line, ret.resolvedTokens);\n        state = ret.ruleStack;\n      }\n\n      this.#stateStack[line + 1] = state;\n      settled =\n        currentChangedRangeEnd !== undefined &&\n        line >= currentChangedRangeEnd &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      line++;\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges?.[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        currentChangedRangeEnd = nextRange[1];\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          settled = false;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          settled = false;\n          continue;\n        }\n      }\n\n      // limit the time of partial tokenize to 1ms\n      if (performance.now() - t > 1) {\n        break;\n      }\n    }\n\n    this.#onDeferTokenize(lines, this.#themeType);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n\n    if (settled || line >= totalLines) {\n      this.stopBackgroundTokenize();\n      return;\n    }\n\n    this.#lastLine = line;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#postTokenizeMessage(jobId);\n  }\n}\n\nexport function tokenizeLine(\n  grammar: IGrammar,\n  colorMap: string[],\n  lineText: string,\n  stateStack: StateStack,\n  timeLimit?: number\n): {\n  ruleStack: StateStack;\n  resolvedTokens: Array<HighlightedToken>;\n} {\n  const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);\n  if (result.stoppedEarly) {\n    console.warn(\n      `[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`\n    );\n  }\n  const rawTokens = result.tokens;\n  const tokensLength = rawTokens.length / 2;\n  const resolvedTokens: Array<HighlightedToken> = [];\n  for (let j = 0; j < tokensLength; j++) {\n    const offset = rawTokens[2 * j];\n    const nextOffset =\n      j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;\n    if (offset === nextOffset) {\n      // should never reach here, skip if happens anyway\n      continue;\n    }\n    const metadata = rawTokens[2 * j + 1];\n    const bg = EncodedTokenMetadata.getForeground(metadata);\n    const fg = colorMap[bg];\n    const tokenText = lineText.slice(offset, nextOffset);\n    resolvedTokens.push([offset, fg, tokenText]);\n  }\n  return {\n    ruleStack: result.ruleStack,\n    resolvedTokens,\n  };\n}\n\nexport function renderLineTokens(\n  tokens: Array<HighlightedToken>,\n  themeType: 'light' | 'dark'\n): (HTMLElement | string)[] {\n  return tokens.map(([char, fg, textContent]) => {\n    if (char === 0 && fg === '') {\n      if (textContent === '') {\n        return h('br');\n      }\n      return textContent;\n    }\n    return h('span', {\n      dataset: {\n        char: char.toString(),\n      },\n      style: `--diffs-token-${themeType}:${fg};`,\n      textContent: textContent,\n    });\n  });\n}\n\n// Shiki special-cases `text` and `ansi` in codeToHast but does not expose grammars.\nfunction isGrammarlessLanguage(languageId: string): boolean {\n  return languageId === 'text' || languageId === 'ansi';\n}\n"],"mappings":";;;;;AAkCA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,sBAAsB;CAE7B;CACA;CACA;CACA;CAIA,aAAa;CACb;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,cAA4B,CAAC,OAAO;CACpC,YAAoB;CACpB,aAAsB;CACtB,YAAqB;CACrB,mBAA2B;CAC3B;CACA,+BAAuC;CACvC,6BAAsC;CAEtC,sBAAsB,SAAS,OAAO,gBAA8B;EAClE,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,CAAC;EACpE,MAAM,UAAU,KAAK,IACnB,eAAe,WAAW,WAAW,eAAe,YACpD,KAAKA,cAAc,SACrB;EACA,IACE,KAAKC,aAAa,KAAA,KAClB,CAAC,sBAAsB,KAAKD,cAAc,UAAU,GACpD;GACA,MAAM,KAAKE,aAAa,aAAa,KAAKF,cAAc,UAAU;GAClE,KAAKC,WAAW,KAAKC,aAAa,YAChC,KAAKF,cAAc,UACrB;EACF;EACA,KAAKG,iBAAiB,OAAO;CAC/B,GAAG,GAAG;CAEN,cAAc,EAAE,WAAkC;EAChD,IAAI,OAAO,SAAS,YAAY,SAAS,MACvC;EAEF,MAAM,EAAE,MAAM,UAAU;EAIxB,IACE,SAAS,cACT,OAAO,UAAU,YACjB,UAAU,KAAKC,kBAEf,KAAKC,oBAAoB,KAAK;CAElC;CAEA,IAAI,YAA8B;EAChC,OAAO,KAAKC;CACd;CAEA,YAAY,EACV,aACA,aACA,cACA,UACA,iBACA,eACA,WACuB;EACvB,MAAM,EACJ,YAAY,UACZ,QAAQ,gBACR,wBAAwB,QACtB;EACJ,KAAKC,kBAAkB,OAAO,WAAW,8BAA8B;EACvE,IAAI,cAAc,UAChB,KAAKD,aAAa,KAAKC,gBAAgB,UAAU,SAAS;OAE1D,KAAKD,aAAa;EAOpB,IAAI,OAAO,UAAU,YAAY,cAAc,UAAU;GACvD,MAAM,WAAW,IAAI,kBAAkB,cAAc;IACnD,KAAK,MAAM,EAAE,MAAM,mBAAmB,WACpC,IACE,SAAS,gBACT,kBAAkB,SACjB,kBAAkB,WAAW,cAAc,WAAW,OAAO,IAC9D;KACA,MAAM,YACJ,iBAAiB,SAAS,IAAI,CAAC,CAAC,gBAAgB,SAC5C,SACA;KACN,KAAKE,iBAAiB,MAAM,YAAY,SAAS;KACjD;IACF;GAEJ,CAAC;GACD,SAAS,QAAQ,SAAS,iBAAiB,EAAE,YAAY,KAAK,CAAC;GAC/D,SAAS,QAAQ,SAAS,MAAM,EAAE,YAAY,KAAK,CAAC;GACpD,KAAKC,YAAY,CACf,iBAAiB,KAAKF,iBAAiB,WAAW,MAAM;IACtD,MAAM,YAAY,EAAE,UAAU,SAAS;IACvC,KAAKC,iBAAiB,MAAM,YAAY,SAAS;GACnD,CAAC,SACK,SAAS,WAAW,CAC5B;EACF;EACA,KAAKN,eAAe;EACpB,KAAKF,gBAAgB;EACrB,KAAKU,yBAAyB;EAC9B,KAAKC,YAAY;EACjB,KAAKC,mBAAmB;EACxB,KAAKC,iBAAiB;EACtB,KAAKC,SAAS,WAAW;EACzB,KAAKC,eAAe;EACpB,KAAKC,YAAY,CAAC;EAClB,KAAKC,UAAU,OAAO,UAAU,WAAW,QAAQ,MAAM,KAAKX,WAAW;CAC3E;CAIA,iBAAiB,WAAmB,WAA6B;EAC/D,KAAKA,aAAa;EAClB,KAAKW,UAAU,SAAS;EACxB,KAAK,uBAAuB;EAC5B,KAAKC,cAAc,CAAC,OAAO;EAC3B,IAAI,KAAKjB,aAAa,KAAA,KAAa,KAAKD,cAAc,YAAY,GAChE,KAAKmB,4BAA4B,CAAC;EAKpC,KAAKN,iBAAiB;CACxB;CAUA,UAAU,aAAoC;EAC5C,MAAM,EAAE,YAAY,UAAU,QAAQ,mBAAmB;EACzD,MAAM,gBACJ,cAAc,WACV,KAAKN,gBAAgB,UACnB,SACA,UACF;EACN,MAAM,gBACJ,OAAO,UAAU,WAAW,QAAQ,MAAM;EAC5C,IACE,kBAAkB,KAAKD,cACvB,kBAAkB,KAAKc,YAEvB;EAEF,KAAKZ,iBAAiB,eAAe,aAAa;CACpD;CAEA,UAAU,WAAmB;EAC3B,KAAKY,aAAa;EAClB,KAAKJ,YAAY,KAAKd,aAAa,SAAS,SAAS,CAAC,CAAC;EACvD,MAAM,EAAE,SAAS,CAAC,MAAM,KAAKA,aAAa,SAAS,SAAS;EAC5D,MAAM,sBAAsB,OAAO;EACnC,MAAM,0BAA0B,OAAO;EACvC,MAAM,mBAAmB,OAAO;EAChC,MAAM,sBAAsB,OAAO;EACnC,MAAM,+BACJ,OAAO;EACT,MAAM,iBAAiB,OAAO;EAC9B,MAAM,iBAAiB,OAAO;EAC9B,MAAM,oBAAoB,OAAO;EACjC,MAAM,kBAAkB,OAAO;EAC/B,KAAKS,UAAU;qCACkB,uBAAuB,uBAAuB;0CACzC,2BAA2B,uBAAuB;iCAC3D,uBAAuB,QAAQ;2CACrB,gCAAgC,QAAQ;kCACjD,oBAAoB,QAAQ;gCAC9B,kBAAkB,QAAQ;gCAC1B,kBAAkB,QAAQ;mCACvB,qBAAqB,QAAQ;iCAC/B,mBAAmB,QAAQ;MACtD;CACJ;CAEA,UAAgB;EACd,KAAK,uBAAuB;EAC5B,KAAKU,uBAAuB;EAC5B,KAAKZ,WAAW,SAAS,YAAY,QAAQ,CAAC;EAC9C,KAAKA,YAAY,KAAA;CACnB;CAIA,SACE,QACA,aACsC;EACtC,KAAKM,eAAe;EACpB,IACE,KAAKd,aAAa,KAAA,KAClB,CAAC,sBAAsB,KAAKD,cAAc,UAAU,GAEpD,MAAM,IAAI,MACR,yBAAyB,KAAKA,cAAc,WAAW,aACzD;EAGF,MAAM,EAAE,cAAc,KAAKA;EAC3B,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,CAAC;EACpE,MAAM,qBACJ,eAAe,WACX,YACA,KAAK,IAAI,eAAe,YAAY,SAAS;EAEnD,MAAM,aAAa,OAAO;EAC1B,MAAM,YAAY,KAAK,IAAI,cAAc,UAAU;EACnD,MAAM,wBACJ,gBAAgB,KAAA,KAChB,eAAe,YACf,OAAO,YAAY,KACnB,aAAa,sBACb,OAAO,WAAW;EACpB,MAAM,uBAAuB,OAAO,cAAc;EAClD,MAAM,0BACJ,wBACA,gBAAgB,KAAA,KAChB,cAAc;EAChB,MAAM,oBAAiD,uBAClD,OAAO,qBAAqB,CAAC,CAAC,YAAY,OAAO,OAAO,CAAC,IAC1D,CAAC,CAAC,YAAY,OAAO,OAAO,CAAC;EACjC,IAAI,mBAAmB;EACvB,IAAI,aAAa;QACV,MAAM,CAAC,YAAY,aAAa,mBACnC,IAAI,aAAa,WACf,mBAAmB,KAAK,IACtB,kBACA,KAAK,IAAI,UAAU,YAAY,CAAC,CAClC;EAAA;EAIN,MAAM,4BACJ,oBAAoB,eACnB,wBAAwB,OAAO,YAAY;EAC9C,IAAI,sBACF,KAAKG,iBAAiB,UAAU;OAC3B;GACL,KAAKe,YAAY,SAAS,KAAK,IAC7B,KAAKA,YAAY,QACjB,aAAa,CACf;GACA,IAAI,gBAAgB,KAAA,KAAa,cAAc,WAC7C,KAAKf,iBAAiB,SAAS;EAEnC;EAEA,IAAI,oBAAoB;EACxB,IAAI,yBAAyB,kBAAkB,kBAAkB,CAAC;EAClE,IAAI;EACJ,IAAI,8BAA8B;EAClC,IAAI,OAAO,uBACP,kBAAkB,kBAAkB,CAAC,KACrC;EACJ,IAAI,UAAU;EACd,MAAM,6BAAmD,IAAI,IAAI;EACjE,MAAM,sBAEU,4CAA4B,IAAI,IAAI,IAAI,KAAA;EACxD,IAAI,wBAAwB,KAAA,KAAa,CAAC,sBAAsB;GAC9D,MAAM,eAAe,KAAK,IACxB,mBAAmB,GACnB,WACA,kBACF;GACA,IAAI,eAAe,YAAY;IAC7B,KAAKA,iBAAiB,YAAY;IAClC,IAAI,gBAAgB;IACpB,IAAI,iBAAiB,KAAKe,YAAY,kBAAkB;IACxD,OAAO,gBAAgB,cAAc,iBAAiB;KACpD,MAAM,WAAW,KAAKI,gBAAgB,eAAe,cAAc;KACnE,iBAAiB,SAAS;KAC1B,oBAAoB,IAAI,eAAe,SAAS,cAAc;IAChE;IACA,IAAI,yBACF,KAAKJ,YAAY,gBAAgB;GAErC;EACF;EAMA,IAAI,QAAQ,KAAKA,YAAY,SAAS;EACtC,OAAO,OAAO,qBAAsB;GAClC,MAAM,oBAAoB,uBACtB,KAAKA,YAAY,OAAO,KACxB,KAAA;GACJ,IAAI,yBACF,KAAKA,YAAY,QAAQ;GAG3B,MAAM,EAAE,gBAAgB,OAAO,cAAc,KAAKI,gBAChD,MACA,KACF;GACA,QAAQ;GAER,IAAI,QAAQ,WACV,WAAW,IAAI,MAAM,cAAc;QAEnC,qBAAqB,IAAI,MAAM,cAAc;GAG/C,IAAI,yBACF,KAAKJ,YAAY,OAAO,KAAK;GAE/B,UACE,QAAQ,0BACR,wBACA,sBAAsB,KAAA,KACtB,MAAM,OAAO,iBAAiB;GAChC,IAAI,SAAS;IACX;IACA,MAAM,YAAY,kBAAkB;IACpC,IAAI,cAAc,KAAA,GAChB;IAEF,IAAI,UAAU,MAAM,oBAAoB;KACtC,sBAAsB,UAAU;KAChC,8BAA8B;KAC9B;IACF;IACA,IAAI,KAAKA,YAAY,UAAU,QAAQ,KAAA,GAAW;KAChD,yBAAyB,UAAU;KACnC;IACF,OAAO;KACL,OAAO,UAAU;KACjB,QAAQ,KAAKA,YAAY,SAAS;KAClC,yBAAyB,UAAU;IACrC;IACA,UAAU;IACV;GACF;GACA;EACF;EAEA,IAAI,yBACF,IAAI,OAAO,oBACT,KAAKA,YAAY,OAAO,KAAK;OAE7B,KAAKA,YAAY,QAAQ;EAI7B,IAAI,wBAAwB,KAAA,KAAa,oBAAoB,OAAO,GAClE,KAAKN,iBAAiB,qBAAqB,KAAKN,UAAU;EAG5D,IAAI,wBAAwB,KAAA,GAC1B,KAAKa,4BACH,qBACA,mBACA,2BACF;OACK,IAAI,CAAC,WAAW,OAAO,WAAW;GACvC,MAAM,iBACJ,yBAAyB,cAAc,YACnC,qBACA,aAAa,aAAa,CAAC,uBACzB,aACA;GACR,KAAKA,4BACH,gBACA,uBAAuB,oBAAoB,KAAA,GAC3C,iBACF;EACF;EAEA,OAAO;CACT;CAEA,mBAAmB,aAAiC;EAClD,KAAKJ,eAAe;EACpB,KAAKQ,oBAAoB,WAAW;CACtC;CAEA,iBAAuB;EACrB,IACE,KAAKtB,aAAa,KAAA,KAClB,CAAC,sBAAsB,KAAKD,cAAc,UAAU,KACpD,KAAKE,aACF,mBAAmB,CAAC,CACpB,SAAS,KAAKF,cAAc,UAAU,GAEzC,KAAKC,WAAW,KAAKC,aAAa,YAChC,KAAKF,cAAc,UACrB;CAEJ;CAEA,yBAA+B;EAC7B,IAAI,KAAKwB,YACP;EAEF,KAAKA,aAAa;EAClB,KAAKC,YAAY;EACjB,KAAKC,YAAY;EACjB,KAAKC,+BAA+B,KAAA;EACpC,KAAKC,+BAA+B;EACpC,KAAKP,uBAAuB;CAC9B;CAEA,0BAAgC;EAC9B,IAAI,KAAKG,cAAc,KAAKC,WAC1B;EAEF,IAAI,KAAKX,QACP,QAAQ,IAAI,iDAAiD,EAC3D,OAAO,KAAKV,iBACd,CAAC;EAEH,KAAKqB,YAAY;CACnB;CAEA,2BAAiC;EAC/B,IACE,KAAKD,cACL,CAAC,KAAKC,aACN,KAAKxB,aAAa,KAAA,KAClB,KAAKyB,YAAY,GAEjB;EAEF,IAAI,KAAKZ,QACP,QAAQ,IAAI,kDAAkD,EAC5D,OAAO,KAAKV,iBACd,CAAC;EAEH,KAAKqB,YAAY;EACjB,KAAKI,qBAAqB,KAAKzB,gBAAgB;CACjD;CAEA,yBAA+B;EAC7B,IAAI,KAAK0B,4BACP;EAEF,WAAW,iBAAiB,WAAW,KAAKC,UAAU;EACtD,KAAKD,6BAA6B;CACpC;CAEA,yBAA+B;EAC7B,IAAI,CAAC,KAAKA,4BACR;EAEF,WAAW,oBAAoB,WAAW,KAAKC,UAAU;EACzD,KAAKD,6BAA6B;CACpC;CAEA,qBAAqB,OAAqB;EAExC,WAAW,YAAY;GAAE,MAAM;GAAY;EAAM,CAAC;CACpD;CAEA,4BACE,WACA,mBACA,oBAAoB,GACd;EACN,IAAI,sBAAsB,KAAK9B,cAAc,UAAU,GACrD;EAGF,MAAM,QAAQ,EAAE,KAAKI;EAErB,IAAI,KAAKU,QACP,QAAQ,IAAI,oDAAoD;GAC9D;GACA;GACA;GACA;EACF,CAAC;EAGH,KAAKU,aAAa;EAClB,KAAKC,YAAY;EACjB,KAAKC,YAAY;EACjB,KAAKC,+BAA+B;EACpC,KAAKC,+BAA+B;EACpC,KAAKI,uBAAuB;EAC5B,KAAKH,qBAAqB,KAAK;CACjC;CAEA,gBACE,MACA,OACgE;EAChE,MAAM,WAAW,KAAK7B,cAAc,YAAY,IAAI;EACpD,IAAI,SAAS,SAAS,KAAKU,wBAAwB;GACjD,QAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,QAC1D;GACA,OAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;IAAQ,CAAC;IAAG;GAAM;EACtD;EACA,IACE,KAAKT,aAAa,KAAA,KAClB,aAAa,MACb,SAAS,KAAK,MAAM,IAEpB,OAAO;GAAE,gBAAgB,CAAC;IAAC;IAAG;IAAI;GAAQ,CAAC;GAAG;EAAM;EAEtD,MAAM,SAAS,aACb,KAAKA,UACL,KAAKe,WACL,UACA,OACA,gBAAgB,mBAClB;EACA,OAAO;GACL,gBAAgB,OAAO;GACvB,OAAO,OAAO;EAChB;CACF;CAEA,iBAAiB,OAAe;EAC9B,MAAM,eAAe,KAAK,IACxB,KAAK,IAAI,GAAG,KAAK,GACjB,KAAKhB,cAAc,SACrB;EACA,IAAI,KAAKkB,YAAY,SAAS,gBAAgB,KAAKjB,aAAa,KAAA,GAC9D;EAEF,IAAI,OAAO,KAAKiB,YAAY,SAAS;EACrC,IAAI,QAAQ,KAAKA,YAAY,SAAS;EACtC,OAAO,OAAO,cAAc,QAAQ;GAClC,KAAKA,YAAY,QAAQ;GACzB,MAAM,WAAW,KAAKlB,cAAc,YAAY,IAAI;GACpD,IACE,SAAS,UAAU,KAAKU,0BACxB,aAAa,MACb,SAAS,KAAK,MAAM,IAEpB,QAAQ,KAAKT,SAAS,cACpB,UACA,OACA,gBAAgB,mBAClB,CAAC,CAAC;EAEN;EACA,KAAKiB,YAAY,QAAQ;CAC3B;CAEA,oBAAoB,OAAe;EACjC,IACE,KAAKM,cACL,KAAKC,aACL,KAAKxB,aAAa,KAAA,KAClB,UAAU,KAAKG,kBAEf;EAGF,MAAM,IAAI,YAAY,IAAI;EAC1B,MAAM,wBAAQ,IAAI,IAAqC;EACvD,MAAM,aAAa,KAAKJ,cAAc;EACtC,MAAM,oBAAoB,KAAK2B;EAE/B,IAAI,OAAO,KAAKD;EAChB,IAAI,QAAQ,KAAKR,YAAY,SAAS;EACtC,IAAI,UAAU;EACd,IAAI,oBAAoB,KAAKU;EAC7B,IAAI,yBAAyB,oBAAoB,kBAAkB,GAAG;EACtE,OAAO,OAAO,aAAc;GAC1B,KAAKV,YAAY,QAAQ;GAEzB,MAAM,oBACJ,2BAA2B,KAAA,IACvB,KAAKA,YAAY,OAAO,KACxB,KAAA;GACN,MAAM,WAAW,KAAKlB,cAAc,YAAY,IAAI;GACpD,IAAI,SAAS,SAAS,KAAKU,wBAAwB;IACjD,QAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,QAC1D;IACA,MAAM,IAAI,MAAM,CAAC;KAAC;KAAG;KAAI;IAAQ,CAAC,CAAC;GACrC,OAAO,IAAI,aAAa,MAAM,SAAS,KAAK,MAAM,IAChD,MAAM,IAAI,MAAM,CAAC;IAAC;IAAG;IAAI;GAAQ,CAAC,CAAC;QAC9B;IACL,MAAM,MAAM,aACV,KAAKT,UACL,KAAKe,WACL,UACA,OACA,gBAAgB,mBAClB;IACA,MAAM,IAAI,MAAM,IAAI,cAAc;IAClC,QAAQ,IAAI;GACd;GAEA,KAAKE,YAAY,OAAO,KAAK;GAC7B,UACE,2BAA2B,KAAA,KAC3B,QAAQ,0BACR,sBAAsB,KAAA,KACtB,MAAM,OAAO,iBAAiB;GAChC;GACA,IAAI,SAAS;IACX;IACA,MAAM,YAAY,oBAAoB;IACtC,IAAI,cAAc,KAAA,GAChB;IAEF,yBAAyB,UAAU;IACnC,IAAI,KAAKA,YAAY,UAAU,QAAQ,KAAA,GACrC,UAAU;SACL;KACL,OAAO,UAAU;KACjB,QAAQ,KAAKA,YAAY,SAAS;KAClC,UAAU;KACV;IACF;GACF;GAGA,IAAI,YAAY,IAAI,IAAI,IAAI,GAC1B;EAEJ;EAEA,KAAKN,iBAAiB,OAAO,KAAKN,UAAU;EAC5C,IAAI,KAAKkB,cAAc,KAAKC,aAAa,UAAU,KAAKrB,kBACtD;EAGF,IAAI,WAAW,QAAQ,YAAY;GACjC,KAAK,uBAAuB;GAC5B;EACF;EAEA,KAAKsB,YAAY;EACjB,KAAKE,+BAA+B;EACpC,KAAKC,qBAAqB,KAAK;CACjC;AACF;AAEA,SAAgB,aACd,SACA,UACA,UACA,YACA,WAIA;CACA,MAAM,SAAS,QAAQ,cAAc,UAAU,YAAY,SAAS;CACpE,IAAI,OAAO,cACT,QAAQ,KACN,oDAAoD,SAAS,UAAU,GAAG,GAAG,GAC/E;CAEF,MAAM,YAAY,OAAO;CACzB,MAAM,eAAe,UAAU,SAAS;CACxC,MAAM,iBAA0C,CAAC;CACjD,KAAK,IAAI,IAAI,GAAG,IAAI,cAAc,KAAK;EACrC,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,aACJ,IAAI,IAAI,eAAe,UAAU,IAAI,IAAI,KAAK,SAAS;EACzD,IAAI,WAAW,YAEb;EAEF,MAAM,WAAW,UAAU,IAAI,IAAI;EAEnC,MAAM,KAAK,SADA,qBAAqB,cAAc,QACzB;EACrB,MAAM,YAAY,SAAS,MAAM,QAAQ,UAAU;EACnD,eAAe,KAAK;GAAC;GAAQ;GAAI;EAAS,CAAC;CAC7C;CACA,OAAO;EACL,WAAW,OAAO;EAClB;CACF;AACF;AAEA,SAAgB,iBACd,QACA,WAC0B;CAC1B,OAAO,OAAO,KAAK,CAAC,MAAM,IAAI,iBAAiB;EAC7C,IAAI,SAAS,KAAK,OAAO,IAAI;GAC3B,IAAI,gBAAgB,IAClB,OAAO,EAAE,IAAI;GAEf,OAAO;EACT;EACA,OAAO,EAAE,QAAQ;GACf,SAAS,EACP,MAAM,KAAK,SAAS,EACtB;GACA,OAAO,iBAAiB,UAAU,GAAG,GAAG;GAC3B;EACf,CAAC;CACH,CAAC;AACH;AAGA,SAAS,sBAAsB,YAA6B;CAC1D,OAAO,eAAe,UAAU,eAAe;AACjD"}

package/dist/editor/utils.d.ts

@@ -13,6 +13,9 @@ declare function clampDomOffset(node: Node, offset: number): number;
 declare function extend<T extends object>(obj: T, attrs: Partial<T>): T;
 declare function debounce<T extends (...args: any[]) => void>(func: T, wait: number): (...args: Parameters<T>) => void;
 declare function round(value: number, precision?: number): number;
+declare function endsWithLineBreak(text: string): boolean;
+declare function createSegmenter(options: Intl.SegmenterOptions): Intl.Segmenter | undefined;
+declare function getGraphemeSegmenter(): Intl.Segmenter | undefined;
 //#endregion
-export { addEventListener, clampDomOffset, debounce, extend, getLineNumberAttr, h, round };
+export { addEventListener, clampDomOffset, createSegmenter, debounce, endsWithLineBreak, extend, getGraphemeSegmenter, getLineNumberAttr, h, round };
 //# sourceMappingURL=utils.d.ts.map