@pierre/diffs 1.3.0-beta.4 → 1.3.0-beta.5

package/dist/editor/textMeasure.js

@@ -11,12 +11,18 @@ var Metrics = class {
 	tabSize = 2;
 	/** Height of the code line. */
 	lineHeight = 20;
+	paddingTop = 0;
 	/** initialize the metrics */
 	init(root) {
 		if (this.#root === root && this.#canvasCtx !== void 0 && this.ch !== -1) return;
 		this.#root = root;
 		this.#canvasCtx ??= document.createElement("canvas").getContext("2d") ?? void 0;
 		if (this.#canvasCtx === void 0) throw new Error("Could not get canvas context");
+		const parent = root.parentElement;
+		if (parent !== null) {
+			const { paddingTop } = getComputedStyle(parent);
+			if (paddingTop.endsWith("px")) this.paddingTop = parseFloat(paddingTop.slice(0, -2));
+		}
 		const { fontSize, fontFamily, tabSize, lineHeight } = getComputedStyle(root);
 		if (lineHeight.endsWith("px")) this.lineHeight = parseFloat(lineHeight.slice(0, -2));
 		else if (fontSize.endsWith("px")) this.lineHeight = round(parseFloat(fontSize.slice(0, -2)) * parseFloat(lineHeight));

package/dist/editor/textMeasure.js.map

@@ -1 +1 @@
-{"version":3,"file":"textMeasure.js","names":["#root","#canvasCtx","#font"],"sources":["../../src/editor/textMeasure.ts"],"sourcesContent":["import { h, round } from './utils';\n\nexport class Metrics {\n  #root?: HTMLElement;\n  #canvasCtx?: CanvasRenderingContext2D;\n  #font?: string;\n\n  /** Width of the '0' character. */\n  ch: number = -1;\n  /** Size of a tab(\\t) character. */\n  tabSize: number = 2;\n  /** Height of the code line. */\n  lineHeight: number = 20;\n\n  /** initialize the metrics */\n  init(root: HTMLElement): void {\n    if (\n      this.#root === root &&\n      this.#canvasCtx !== undefined &&\n      this.ch !== -1\n    ) {\n      // already initialized\n      return;\n    }\n\n    this.#root = root;\n    this.#canvasCtx ??=\n      document.createElement('canvas').getContext('2d') ?? undefined;\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Could not get canvas context');\n    }\n\n    const { fontSize, fontFamily, tabSize, lineHeight } =\n      getComputedStyle(root);\n    if (lineHeight.endsWith('px')) {\n      this.lineHeight = parseFloat(lineHeight.slice(0, -2));\n    } else if (fontSize.endsWith('px')) {\n      this.lineHeight = round(\n        parseFloat(fontSize.slice(0, -2)) * parseFloat(lineHeight)\n      );\n    }\n    const font = fontSize + ' ' + fontFamily;\n    if (this.#font !== font || this.ch === -1) {\n      this.#font = font;\n      this.#canvasCtx.font = font;\n      this.ch = this.canvasMeasureTextWidth('0');\n    }\n    this.tabSize = parseInt(tabSize, 10);\n  }\n\n  /** measure the width of the text */\n  measureTextWidth(text: string): number {\n    const textWithExpandedTabs = text.replaceAll(\n      '\\t',\n      ' '.repeat(this.tabSize)\n    );\n    if (needsDomTextMeasurement(textWithExpandedTabs)) {\n      return this.domMeasureTextWidth(textWithExpandedTabs);\n    }\n    return this.canvasMeasureTextWidth(textWithExpandedTabs);\n  }\n\n  /** measure the width of the text using the canvas measureText API */\n  canvasMeasureTextWidth(text: string): number {\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    return round(this.#canvasCtx.measureText(text).width);\n  }\n\n  /**\n   * measure the width of the text using the DOM\n   * this is slow because it cause a reflow, use it for non-ascii text\n   */\n  domMeasureTextWidth(text: string): number {\n    if (this.#root === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    const measureEl = h(\n      'span',\n      {\n        style: {\n          position: 'absolute',\n          top: '0',\n          left: '0',\n          visibility: 'hidden',\n          pointerEvents: 'none',\n          whiteSpace: 'pre',\n          font: 'inherit',\n        },\n        textContent: text,\n      },\n      this.#root\n    );\n    try {\n      return measureEl.getBoundingClientRect().width;\n    } finally {\n      measureEl.remove();\n    }\n  }\n}\n\n/** Check if the text needs DOM text measurement. */\nexport function needsDomTextMeasurement(text: string): boolean {\n  for (let i = 0; i < text.length; i++) {\n    const code = text.charCodeAt(i);\n    if (\n      (code >= 0xd800 && code <= 0xdfff) ||\n      code === 0x200d ||\n      code === 0xfe0e ||\n      code === 0xfe0f\n    ) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/** snap the text offset to the Unicode boundary */\nexport function snapTextOffsetToUnicodeBoundary(\n  text: string,\n  offset: number\n): number {\n  const boundedOffset = Math.max(0, Math.min(offset, text.length));\n  if (\n    boundedOffset === 0 ||\n    boundedOffset === text.length ||\n    !needsDomTextMeasurement(text)\n  ) {\n    return boundedOffset;\n  }\n  // Avoid measuring a caret position inside one visual emoji/grapheme.\n  // Browser caret movement can report offsets around UTF-16 surrogate\n  // pairs and emoji joiners; measuring a partial sequence gives a\n  // replacement-glyph width.\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    const segmentStart = segment.index;\n    const segmentEnd = segmentStart + segment.segment.length;\n    if (boundedOffset > segmentStart && boundedOffset < segmentEnd) {\n      return segmentEnd;\n    }\n    if (boundedOffset <= segmentStart) {\n      break;\n    }\n  }\n  return boundedOffset;\n}\n\n/** get the offsets of the Unicode grapheme clusters in the text */\nexport function getUnicodeMeasurementOffsets(\n  text: string\n): number[] | undefined {\n  if (!needsDomTextMeasurement(text)) {\n    return undefined;\n  }\n  const offsets = [0];\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    offsets.push(segment.index + segment.segment.length);\n  }\n  return offsets;\n}\n\n/** get the number of columns of the ASCII text */\nexport function getExpandedAsciiTextColumns(\n  text: string,\n  tabSize: number\n): number {\n  let columns = 0;\n  for (let i = 0; i < text.length; i++) {\n    if (text.charCodeAt(i) > 127) {\n      return -1;\n    }\n    columns += text.charCodeAt(i) === /* '\\t' */ 9 ? tabSize : 1;\n  }\n  return columns;\n}\n"],"mappings":";;;AAEA,IAAa,UAAb,MAAqB;CACnB;CACA;CACA;;CAGA,KAAa;;CAEb,UAAkB;;CAElB,aAAqB;;CAGrB,KAAK,MAAyB;AAC5B,MACE,MAAKA,SAAU,QACf,MAAKC,cAAe,UACpB,KAAK,OAAO,GAGZ;AAGF,QAAKD,OAAQ;AACb,QAAKC,cACH,SAAS,cAAc,SAAS,CAAC,WAAW,KAAK,IAAI;AACvD,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,+BAA+B;EAGjD,MAAM,EAAE,UAAU,YAAY,SAAS,eACrC,iBAAiB,KAAK;AACxB,MAAI,WAAW,SAAS,KAAK,CAC3B,MAAK,aAAa,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;WAC5C,SAAS,SAAS,KAAK,CAChC,MAAK,aAAa,MAChB,WAAW,SAAS,MAAM,GAAG,GAAG,CAAC,GAAG,WAAW,WAAW,CAC3D;EAEH,MAAM,OAAO,WAAW,MAAM;AAC9B,MAAI,MAAKC,SAAU,QAAQ,KAAK,OAAO,IAAI;AACzC,SAAKA,OAAQ;AACb,SAAKD,UAAW,OAAO;AACvB,QAAK,KAAK,KAAK,uBAAuB,IAAI;;AAE5C,OAAK,UAAU,SAAS,SAAS,GAAG;;;CAItC,iBAAiB,MAAsB;EACrC,MAAM,uBAAuB,KAAK,WAChC,KACA,IAAI,OAAO,KAAK,QAAQ,CACzB;AACD,MAAI,wBAAwB,qBAAqB,CAC/C,QAAO,KAAK,oBAAoB,qBAAqB;AAEvD,SAAO,KAAK,uBAAuB,qBAAqB;;;CAI1D,uBAAuB,MAAsB;AAC3C,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,0BAA0B;AAE5C,SAAO,MAAM,MAAKA,UAAW,YAAY,KAAK,CAAC,MAAM;;;;;;CAOvD,oBAAoB,MAAsB;AACxC,MAAI,MAAKD,SAAU,OACjB,OAAM,IAAI,MAAM,0BAA0B;EAE5C,MAAM,YAAY,EAChB,QACA;GACE,OAAO;IACL,UAAU;IACV,KAAK;IACL,MAAM;IACN,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,MAAM;IACP;GACD,aAAa;GACd,EACD,MAAKA,KACN;AACD,MAAI;AACF,UAAO,UAAU,uBAAuB,CAAC;YACjC;AACR,aAAU,QAAQ;;;;;AAMxB,SAAgB,wBAAwB,MAAuB;AAC7D,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,OAAO,KAAK,WAAW,EAAE;AAC/B,MACG,QAAQ,SAAU,QAAQ,SAC3B,SAAS,QACT,SAAS,SACT,SAAS,MAET,QAAO;;AAGX,QAAO;;;AAIT,SAAgB,gCACd,MACA,QACQ;CACR,MAAM,gBAAgB,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,OAAO,CAAC;AAChE,KACE,kBAAkB,KAClB,kBAAkB,KAAK,UACvB,CAAC,wBAAwB,KAAK,CAE9B,QAAO;CAMT,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,EAAE;EAC7C,MAAM,eAAe,QAAQ;EAC7B,MAAM,aAAa,eAAe,QAAQ,QAAQ;AAClD,MAAI,gBAAgB,gBAAgB,gBAAgB,WAClD,QAAO;AAET,MAAI,iBAAiB,aACnB;;AAGJ,QAAO;;;AAIT,SAAgB,6BACd,MACsB;AACtB,KAAI,CAAC,wBAAwB,KAAK,CAChC;CAEF,MAAM,UAAU,CAAC,EAAE;CACnB,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,CAC3C,SAAQ,KAAK,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAEtD,QAAO;;;AAIT,SAAgB,4BACd,MACA,SACQ;CACR,IAAI,UAAU;AACd,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,MAAI,KAAK,WAAW,EAAE,GAAG,IACvB,QAAO;AAET,aAAW,KAAK,WAAW,EAAE,KAAgB,IAAI,UAAU;;AAE7D,QAAO"}
+{"version":3,"file":"textMeasure.js","names":["#root","#canvasCtx","#font"],"sources":["../../src/editor/textMeasure.ts"],"sourcesContent":["import { h, round } from './utils';\n\nexport class Metrics {\n  #root?: HTMLElement;\n  #canvasCtx?: CanvasRenderingContext2D;\n  #font?: string;\n\n  /** Width of the '0' character. */\n  ch: number = -1;\n  /** Size of a tab(\\t) character. */\n  tabSize: number = 2;\n  /** Height of the code line. */\n  lineHeight: number = 20;\n\n  paddingTop: number = 0;\n\n  /** initialize the metrics */\n  init(root: HTMLElement): void {\n    if (\n      this.#root === root &&\n      this.#canvasCtx !== undefined &&\n      this.ch !== -1\n    ) {\n      // already initialized\n      return;\n    }\n\n    this.#root = root;\n    this.#canvasCtx ??=\n      document.createElement('canvas').getContext('2d') ?? undefined;\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Could not get canvas context');\n    }\n\n    const parent = root.parentElement;\n    if (parent !== null) {\n      const { paddingTop } = getComputedStyle(parent);\n      if (paddingTop.endsWith('px')) {\n        this.paddingTop = parseFloat(paddingTop.slice(0, -2));\n      }\n    }\n\n    const { fontSize, fontFamily, tabSize, lineHeight } =\n      getComputedStyle(root);\n    if (lineHeight.endsWith('px')) {\n      this.lineHeight = parseFloat(lineHeight.slice(0, -2));\n    } else if (fontSize.endsWith('px')) {\n      this.lineHeight = round(\n        parseFloat(fontSize.slice(0, -2)) * parseFloat(lineHeight)\n      );\n    }\n    const font = fontSize + ' ' + fontFamily;\n    if (this.#font !== font || this.ch === -1) {\n      this.#font = font;\n      this.#canvasCtx.font = font;\n      this.ch = this.canvasMeasureTextWidth('0');\n    }\n    this.tabSize = parseInt(tabSize, 10);\n  }\n\n  /** measure the width of the text */\n  measureTextWidth(text: string): number {\n    const textWithExpandedTabs = text.replaceAll(\n      '\\t',\n      ' '.repeat(this.tabSize)\n    );\n    if (needsDomTextMeasurement(textWithExpandedTabs)) {\n      return this.domMeasureTextWidth(textWithExpandedTabs);\n    }\n    return this.canvasMeasureTextWidth(textWithExpandedTabs);\n  }\n\n  /** measure the width of the text using the canvas measureText API */\n  canvasMeasureTextWidth(text: string): number {\n    if (this.#canvasCtx === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    return round(this.#canvasCtx.measureText(text).width);\n  }\n\n  /**\n   * measure the width of the text using the DOM\n   * this is slow because it cause a reflow, use it for non-ascii text\n   */\n  domMeasureTextWidth(text: string): number {\n    if (this.#root === undefined) {\n      throw new Error('Metrics not initialized');\n    }\n    const measureEl = h(\n      'span',\n      {\n        style: {\n          position: 'absolute',\n          top: '0',\n          left: '0',\n          visibility: 'hidden',\n          pointerEvents: 'none',\n          whiteSpace: 'pre',\n          font: 'inherit',\n        },\n        textContent: text,\n      },\n      this.#root\n    );\n    try {\n      return measureEl.getBoundingClientRect().width;\n    } finally {\n      measureEl.remove();\n    }\n  }\n}\n\n/** Check if the text needs DOM text measurement. */\nexport function needsDomTextMeasurement(text: string): boolean {\n  for (let i = 0; i < text.length; i++) {\n    const code = text.charCodeAt(i);\n    if (\n      (code >= 0xd800 && code <= 0xdfff) ||\n      code === 0x200d ||\n      code === 0xfe0e ||\n      code === 0xfe0f\n    ) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/** snap the text offset to the Unicode boundary */\nexport function snapTextOffsetToUnicodeBoundary(\n  text: string,\n  offset: number\n): number {\n  const boundedOffset = Math.max(0, Math.min(offset, text.length));\n  if (\n    boundedOffset === 0 ||\n    boundedOffset === text.length ||\n    !needsDomTextMeasurement(text)\n  ) {\n    return boundedOffset;\n  }\n  // Avoid measuring a caret position inside one visual emoji/grapheme.\n  // Browser caret movement can report offsets around UTF-16 surrogate\n  // pairs and emoji joiners; measuring a partial sequence gives a\n  // replacement-glyph width.\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    const segmentStart = segment.index;\n    const segmentEnd = segmentStart + segment.segment.length;\n    if (boundedOffset > segmentStart && boundedOffset < segmentEnd) {\n      return segmentEnd;\n    }\n    if (boundedOffset <= segmentStart) {\n      break;\n    }\n  }\n  return boundedOffset;\n}\n\n/** get the offsets of the Unicode grapheme clusters in the text */\nexport function getUnicodeMeasurementOffsets(\n  text: string\n): number[] | undefined {\n  if (!needsDomTextMeasurement(text)) {\n    return undefined;\n  }\n  const offsets = [0];\n  const segmenter = new Intl.Segmenter(undefined, {\n    granularity: 'grapheme',\n  });\n  for (const segment of segmenter.segment(text)) {\n    offsets.push(segment.index + segment.segment.length);\n  }\n  return offsets;\n}\n\n/** get the number of columns of the ASCII text */\nexport function getExpandedAsciiTextColumns(\n  text: string,\n  tabSize: number\n): number {\n  let columns = 0;\n  for (let i = 0; i < text.length; i++) {\n    if (text.charCodeAt(i) > 127) {\n      return -1;\n    }\n    columns += text.charCodeAt(i) === /* '\\t' */ 9 ? tabSize : 1;\n  }\n  return columns;\n}\n"],"mappings":";;;AAEA,IAAa,UAAb,MAAqB;CACnB;CACA;CACA;;CAGA,KAAa;;CAEb,UAAkB;;CAElB,aAAqB;CAErB,aAAqB;;CAGrB,KAAK,MAAyB;AAC5B,MACE,MAAKA,SAAU,QACf,MAAKC,cAAe,UACpB,KAAK,OAAO,GAGZ;AAGF,QAAKD,OAAQ;AACb,QAAKC,cACH,SAAS,cAAc,SAAS,CAAC,WAAW,KAAK,IAAI;AACvD,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,+BAA+B;EAGjD,MAAM,SAAS,KAAK;AACpB,MAAI,WAAW,MAAM;GACnB,MAAM,EAAE,eAAe,iBAAiB,OAAO;AAC/C,OAAI,WAAW,SAAS,KAAK,CAC3B,MAAK,aAAa,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;;EAIzD,MAAM,EAAE,UAAU,YAAY,SAAS,eACrC,iBAAiB,KAAK;AACxB,MAAI,WAAW,SAAS,KAAK,CAC3B,MAAK,aAAa,WAAW,WAAW,MAAM,GAAG,GAAG,CAAC;WAC5C,SAAS,SAAS,KAAK,CAChC,MAAK,aAAa,MAChB,WAAW,SAAS,MAAM,GAAG,GAAG,CAAC,GAAG,WAAW,WAAW,CAC3D;EAEH,MAAM,OAAO,WAAW,MAAM;AAC9B,MAAI,MAAKC,SAAU,QAAQ,KAAK,OAAO,IAAI;AACzC,SAAKA,OAAQ;AACb,SAAKD,UAAW,OAAO;AACvB,QAAK,KAAK,KAAK,uBAAuB,IAAI;;AAE5C,OAAK,UAAU,SAAS,SAAS,GAAG;;;CAItC,iBAAiB,MAAsB;EACrC,MAAM,uBAAuB,KAAK,WAChC,KACA,IAAI,OAAO,KAAK,QAAQ,CACzB;AACD,MAAI,wBAAwB,qBAAqB,CAC/C,QAAO,KAAK,oBAAoB,qBAAqB;AAEvD,SAAO,KAAK,uBAAuB,qBAAqB;;;CAI1D,uBAAuB,MAAsB;AAC3C,MAAI,MAAKA,cAAe,OACtB,OAAM,IAAI,MAAM,0BAA0B;AAE5C,SAAO,MAAM,MAAKA,UAAW,YAAY,KAAK,CAAC,MAAM;;;;;;CAOvD,oBAAoB,MAAsB;AACxC,MAAI,MAAKD,SAAU,OACjB,OAAM,IAAI,MAAM,0BAA0B;EAE5C,MAAM,YAAY,EAChB,QACA;GACE,OAAO;IACL,UAAU;IACV,KAAK;IACL,MAAM;IACN,YAAY;IACZ,eAAe;IACf,YAAY;IACZ,MAAM;IACP;GACD,aAAa;GACd,EACD,MAAKA,KACN;AACD,MAAI;AACF,UAAO,UAAU,uBAAuB,CAAC;YACjC;AACR,aAAU,QAAQ;;;;;AAMxB,SAAgB,wBAAwB,MAAuB;AAC7D,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,OAAO,KAAK,WAAW,EAAE;AAC/B,MACG,QAAQ,SAAU,QAAQ,SAC3B,SAAS,QACT,SAAS,SACT,SAAS,MAET,QAAO;;AAGX,QAAO;;;AAIT,SAAgB,gCACd,MACA,QACQ;CACR,MAAM,gBAAgB,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,OAAO,CAAC;AAChE,KACE,kBAAkB,KAClB,kBAAkB,KAAK,UACvB,CAAC,wBAAwB,KAAK,CAE9B,QAAO;CAMT,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,EAAE;EAC7C,MAAM,eAAe,QAAQ;EAC7B,MAAM,aAAa,eAAe,QAAQ,QAAQ;AAClD,MAAI,gBAAgB,gBAAgB,gBAAgB,WAClD,QAAO;AAET,MAAI,iBAAiB,aACnB;;AAGJ,QAAO;;;AAIT,SAAgB,6BACd,MACsB;AACtB,KAAI,CAAC,wBAAwB,KAAK,CAChC;CAEF,MAAM,UAAU,CAAC,EAAE;CACnB,MAAM,YAAY,IAAI,KAAK,UAAU,QAAW,EAC9C,aAAa,YACd,CAAC;AACF,MAAK,MAAM,WAAW,UAAU,QAAQ,KAAK,CAC3C,SAAQ,KAAK,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAEtD,QAAO;;;AAIT,SAAgB,4BACd,MACA,SACQ;CACR,IAAI,UAAU;AACd,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,MAAI,KAAK,WAAW,EAAE,GAAG,IACvB,QAAO;AAET,aAAW,KAAK,WAAW,EAAE,KAAgB,IAAI,UAAU;;AAE7D,QAAO"}

package/dist/editor/tokenzier.js

@@ -28,7 +28,7 @@ var EditorTokenizer = class EditorTokenizer {
 	#prebuildStateStack = debounce(async (renderRange) => {
 		const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};
 		const endLine = Math.min(totalLines === Infinity ? Infinity : startingLine + totalLines, this.#textDocument.lineCount);
-		if (this.#grammar === void 0) {
+		if (this.#grammar === void 0 && !isGrammarlessLanguage(this.#textDocument.languageId)) {
 			await this.#highlighter.loadLanguage(this.#textDocument.languageId);
 			this.#grammar = this.#highlighter.getLanguage(this.#textDocument.languageId);
 		}
@@ -68,7 +68,7 @@ var EditorTokenizer = class EditorTokenizer {
 		this.#setStyle = setStyle;
 		this.#onDeferTokenize = onDeferTokenize;
 		this.#debug = __debug ?? false;
-		if (highlighter.getLoadedLanguages().includes(textDocument.languageId)) this.#grammar = highlighter.getLanguage(textDocument.languageId);
+		if (!isGrammarlessLanguage(textDocument.languageId) && highlighter.getLoadedLanguages().includes(textDocument.languageId)) this.#grammar = highlighter.getLanguage(textDocument.languageId);
 		this.#colorMap = [];
 		this.#setTheme(typeof theme === "string" ? theme : theme[this.#themeType]);
 	}
@@ -109,13 +109,13 @@ var EditorTokenizer = class EditorTokenizer {
     }`);
 	}
 	cleanUp() {
-		this.#detachMessageListener();
 		this.stopBackgroundTokenize();
+		this.#detachMessageListener();
 		this.#disposes?.forEach((dispose) => dispose());
 		this.#disposes = void 0;
 	}
 	tokenize(change, renderRange) {
-		if (this.#grammar === void 0) throw new Error("Grammar not loaded");
+		if (this.#grammar === void 0 && !isGrammarlessLanguage(this.#textDocument.languageId)) throw new Error("Grammar not loaded");
 		const { lineCount } = this.#textDocument;
 		const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};
 		const renderRangeEndLine = totalLines === Infinity ? lineCount : Math.min(startingLine + totalLines, lineCount);
@@ -239,6 +239,7 @@ var EditorTokenizer = class EditorTokenizer {
 		});
 	}
 	#scheduleBackgroundTokenize(startLine, changedLineRanges, changedRangeIndex = 0) {
+		if (isGrammarlessLanguage(this.#textDocument.languageId)) return;
 		const jobId = ++this.#backgroundJobId;
 		if (this.#debug) console.log("[diffs/editor] background tokenization scheduled", {
 			jobId,
@@ -255,7 +256,6 @@ var EditorTokenizer = class EditorTokenizer {
 		this.#postTokenizeMessage(jobId);
 	}
 	#tokenizeLineAt(line, state) {
-		if (this.#grammar === void 0) throw new Error("Grammar not loaded");
 		const lineText = this.#textDocument.getLineText(line);
 		if (lineText.length > this.#tokenizeMaxLineLength) {
 			console.warn(`[diffs] Line(${line}) too long to tokenize: ${lineText.length}`);
@@ -268,7 +268,7 @@ var EditorTokenizer = class EditorTokenizer {
 				state
 			};
 		}
-		if (lineText === "" || lineText.trim() === "") return {
+		if (this.#grammar === void 0 || lineText === "" || lineText.trim() === "") return {
 			resolvedTokens: [[
 				0,
 				"",
@@ -392,6 +392,9 @@ function renderLineTokens(tokens, themeType) {
 		});
 	});
 }
+function isGrammarlessLanguage(languageId) {
+	return languageId === "text" || languageId === "ansi";
+}
 
 //#endregion
 export { EditorTokenizer, renderLineTokens, tokenizeLine };

package/dist/editor/tokenzier.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenzier.js","names":["#textDocument","#grammar","#highlighter","#buildStateStack","#backgroundJobId","#backgroundTokenize","#themeType","#mediaQueryList","themeType","#emitThemeChange","#disposes","#tokenizeMaxLineLength","#setStyle","#onDeferTokenize","#debug","#colorMap","#setTheme","#stateStack","#scheduleBackgroundTokenize","#detachMessageListener","changedLineRanges: readonly [number, number][]","backgroundStartLine: number | undefined","dirtyLines: Map<number, Array<HighlightedToken>>","offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined","#tokenizeLineAt","#prebuildStateStack","#isStopped","#isPaused","#lastLine","#backgroundChangedLineRanges","#backgroundChangedRangeIndex","#postTokenizeMessage","#isMessageListenerAttached","#onMessage","#attachMessageListener","resolvedTokens: Array<HighlightedToken>"],"sources":["../../src/editor/tokenzier.ts"],"sourcesContent":["import {\n  EncodedTokenMetadata,\n  type IGrammar,\n  INITIAL,\n  type StateStack,\n} from 'shiki/textmate';\n\nimport { DEFAULT_THEMES } from '../constants';\nimport type {\n  BaseCodeOptions,\n  DiffsHighlighter,\n  HighlightedToken,\n  RenderRange,\n} from '../types';\nimport type { TextDocument, TextDocumentChange } from './textDocument';\nimport { addEventListener, debounce, h } from './utils';\n\nexport interface EditorTokenizerProps {\n  highlighter: DiffsHighlighter;\n  textDocument: TextDocument<unknown>;\n  codeOptions: BaseCodeOptions;\n  setStyle: (style: string) => void;\n  onDeferTokenize: (\n    lines: Map<number, Array<HighlightedToken>>,\n    themeType: 'dark' | 'light'\n  ) => void;\n  __debug?: boolean;\n}\n\n/** Stoppable code tokenizer for the editor */\nexport class EditorTokenizer {\n  static TOKENIZE_TIME_LIMIT = 500;\n\n  #highlighter: DiffsHighlighter;\n  #grammar: IGrammar | undefined;\n  #mediaQueryList: MediaQueryList;\n  #themeType: 'light' | 'dark';\n  #colorMap: string[];\n  #textDocument: TextDocument<unknown>;\n  #tokenizeMaxLineLength: number;\n  #setStyle: EditorTokenizerProps['setStyle'];\n  #onDeferTokenize: EditorTokenizerProps['onDeferTokenize'];\n  #debug: boolean;\n  #disposes?: (() => void)[];\n\n  // state\n  #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index\n  #lastLine: number = -1;\n  #isStopped: boolean = true;\n  #isPaused: boolean = false;\n  #backgroundJobId: number = 0;\n  #backgroundChangedLineRanges: readonly [number, number][] | undefined;\n  #backgroundChangedRangeIndex: number = 0;\n  #isMessageListenerAttached: boolean = false;\n\n  #prebuildStateStack = debounce(async (renderRange?: RenderRange) => {\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const endLine = Math.min(\n      totalLines === Infinity ? Infinity : startingLine + totalLines,\n      this.#textDocument.lineCount\n    );\n    if (this.#grammar === undefined) {\n      await this.#highlighter.loadLanguage(this.#textDocument.languageId);\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n    this.#buildStateStack(endLine);\n  }, 500);\n\n  #onMessage = ({ data }: MessageEvent<unknown>) => {\n    if (typeof data !== 'object' || data === null) {\n      return;\n    }\n    const { type, jobId } = data as {\n      type?: unknown;\n      jobId?: unknown;\n    };\n    if (\n      type === 'tokenize' &&\n      typeof jobId === 'number' &&\n      jobId === this.#backgroundJobId\n    ) {\n      this.#backgroundTokenize(jobId);\n    }\n  };\n\n  get themeType(): 'light' | 'dark' {\n    return this.#themeType;\n  }\n\n  constructor({\n    codeOptions,\n    highlighter,\n    textDocument,\n    setStyle,\n    onDeferTokenize,\n    __debug,\n  }: EditorTokenizerProps) {\n    const {\n      themeType = 'system',\n      theme = DEFAULT_THEMES,\n      tokenizeMaxLineLength = 1000,\n    } = codeOptions;\n    this.#mediaQueryList = window.matchMedia('(prefers-color-scheme: dark)');\n    if (themeType === 'system') {\n      this.#themeType = this.#mediaQueryList.matches ? 'dark' : 'light';\n    } else {\n      this.#themeType = themeType;\n    }\n    if (typeof theme !== 'string') {\n      const observer = new MutationObserver((mutations) => {\n        for (const { type, attributeName } of mutations) {\n          if (\n            type === 'attributes' &&\n            attributeName !== null &&\n            (attributeName === 'class' || attributeName.startsWith('data-'))\n          ) {\n            const themeType =\n              getComputedStyle(document.body).colorScheme === 'dark'\n                ? 'dark'\n                : 'light';\n            this.#emitThemeChange(theme[themeType], themeType);\n            break;\n          }\n        }\n      });\n      observer.observe(document.documentElement, { attributes: true });\n      observer.observe(document.body, { attributes: true });\n      this.#disposes = [\n        addEventListener(this.#mediaQueryList, 'change', (e) => {\n          const themeType = e.matches ? 'dark' : 'light';\n          this.#emitThemeChange(theme[themeType], themeType);\n        }),\n        () => observer.disconnect(),\n      ];\n    }\n    this.#highlighter = highlighter;\n    this.#textDocument = textDocument;\n    this.#tokenizeMaxLineLength = tokenizeMaxLineLength;\n    this.#setStyle = setStyle;\n    this.#onDeferTokenize = onDeferTokenize;\n    this.#debug = __debug ?? false;\n    if (highlighter.getLoadedLanguages().includes(textDocument.languageId)) {\n      this.#grammar = highlighter.getLanguage(textDocument.languageId);\n    }\n    this.#colorMap = [];\n    this.#setTheme(typeof theme === 'string' ? theme : theme[this.#themeType]);\n  }\n\n  // By default, diffs components support dual themes, but the tokenizer only renders\n  // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document.\n  #emitThemeChange(themeName: string, themeType: 'light' | 'dark') {\n    this.#themeType = themeType;\n    this.#setTheme(themeName);\n    this.stopBackgroundTokenize();\n    this.#stateStack = [INITIAL];\n    if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) {\n      this.#scheduleBackgroundTokenize(0);\n    }\n  }\n\n  #setTheme(themeName: string) {\n    this.#colorMap = this.#highlighter.setTheme(themeName).colorMap;\n    const { colors = {} } = this.#highlighter.getTheme(themeName);\n    const selectionBackground = colors['editor.selectionBackground'];\n    const lineHighlightBackground = colors['editor.lineHighlightBackground'];\n    const gutterForeground = colors['editorLineNumber.foreground'];\n    const gutterActiveForeground = colors['editorLineNumber.activeForeground'];\n    const cursorForeground = colors['editorCursor.foreground'];\n    const findMatchBackground = colors['editor.findMatchBackground'];\n    const findMatchHighlightBackground =\n      colors['editor.findMatchHighlightBackground'];\n    const hintForeground = colors['editorHint.foreground'];\n    const infoForeground = colors['editorInfo.foreground'];\n    const warningForeground = colors['editorWarning.foreground'];\n    const errorForeground = colors['editorError.foreground'];\n    this.#setStyle(`:host {\n      --diffs-editor-selection-bg: ${selectionBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-highlight-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-number-fg: ${gutterForeground ?? 'var(--diffs-fg-number)'};\n      --diffs-editor-line-number-active-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg, var(--diffs-bg))'};\n      --diffs-editor-line-number-active-fg: ${gutterActiveForeground ?? 'var(--diffs-selection-number-fg)'};\n      --diffs-editor-match-bg: ${findMatchBackground ?? 'unset'};\n      --diffs-editor-match-highlight-bg: ${findMatchHighlightBackground ?? 'unset'};\n      --diffs-editor-cursor-fg: ${cursorForeground ?? 'unset'};\n      --diffs-editor-hint-fg: ${hintForeground ?? 'unset'};\n      --diffs-editor-info-fg: ${infoForeground ?? 'unset'};\n      --diffs-editor-warning-fg: ${warningForeground ?? 'unset'};\n      --diffs-editor-error-fg: ${errorForeground ?? 'unset'};\n    }`);\n  }\n\n  cleanUp(): void {\n    this.#detachMessageListener();\n    this.stopBackgroundTokenize();\n    this.#disposes?.forEach((dispose) => dispose());\n    this.#disposes = undefined;\n  }\n\n  // to use `tokenize`, call `prebuildStateStackMap` first to prebuild\n  // the state stack map for the given render range.\n  tokenize(\n    change: TextDocumentChange,\n    renderRange?: RenderRange\n  ): Map<number, Array<HighlightedToken>> {\n    if (this.#grammar === undefined) {\n      throw new Error('Grammar not loaded');\n    }\n\n    const { lineCount } = this.#textDocument;\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const renderRangeEndLine =\n      totalLines === Infinity\n        ? lineCount\n        : Math.min(startingLine + totalLines, lineCount);\n\n    const dirtyStart = change.startLine;\n    const viewStart = Math.max(startingLine, dirtyStart);\n    const crossesRenderRangeEnd =\n      renderRange !== undefined &&\n      totalLines !== Infinity &&\n      change.lineDelta > 0 &&\n      dirtyStart < renderRangeEndLine &&\n      change.endLine >= renderRangeEndLine;\n    const canReuseCachedStates = change.lineDelta === 0;\n    const canCacheTokenizedStates =\n      canReuseCachedStates ||\n      renderRange === undefined ||\n      dirtyStart >= viewStart;\n    const changedLineRanges: readonly [number, number][] = canReuseCachedStates\n      ? (change.changedLineRanges ?? [[dirtyStart, change.endLine]])\n      : [[dirtyStart, change.endLine]];\n    let offscreenSyncEnd = -1;\n    if (dirtyStart < viewStart) {\n      for (const [rangeStart, rangeEnd] of changedLineRanges) {\n        if (rangeStart < viewStart) {\n          offscreenSyncEnd = Math.max(\n            offscreenSyncEnd,\n            Math.min(rangeEnd, viewStart - 1)\n          );\n        }\n      }\n    }\n    const shouldFlushOffscreenLines =\n      offscreenSyncEnd >= dirtyStart &&\n      (canReuseCachedStates || change.lineDelta < 0);\n    if (canReuseCachedStates) {\n      this.#buildStateStack(dirtyStart);\n    } else {\n      this.#stateStack.length = Math.min(\n        this.#stateStack.length,\n        dirtyStart + 1\n      );\n      if (renderRange === undefined || dirtyStart >= viewStart) {\n        this.#buildStateStack(viewStart);\n      }\n    }\n\n    let changedRangeIndex = 0;\n    let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];\n    let backgroundStartLine: number | undefined;\n    let backgroundChangedRangeIndex = 0;\n    let line = canReuseCachedStates\n      ? changedLineRanges[changedRangeIndex][0]\n      : viewStart;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    const dirtyLines: Map<number, Array<HighlightedToken>> = new Map();\n    const offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined = shouldFlushOffscreenLines ? new Map() : undefined;\n    if (offscreenDirtyLines !== undefined && !canReuseCachedStates) {\n      const offscreenEnd = Math.min(\n        offscreenSyncEnd + 1,\n        viewStart,\n        renderRangeEndLine\n      );\n      if (offscreenEnd > dirtyStart) {\n        this.#buildStateStack(offscreenEnd);\n        let offscreenLine = dirtyStart;\n        let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;\n        for (; offscreenLine < offscreenEnd; offscreenLine++) {\n          const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);\n          offscreenState = resolved.state;\n          offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);\n        }\n        if (canCacheTokenizedStates) {\n          this.#stateStack[offscreenEnd] = offscreenState;\n        }\n      }\n    }\n    for (; line < renderRangeEndLine; ) {\n      const previousNextState = canReuseCachedStates\n        ? this.#stateStack[line + 1]\n        : undefined;\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line] = state;\n      }\n\n      const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(\n        line,\n        state\n      );\n      state = nextState;\n\n      if (line >= viewStart) {\n        dirtyLines.set(line, resolvedTokens);\n      } else {\n        offscreenDirtyLines?.set(line, resolvedTokens);\n      }\n\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line + 1] = state;\n      }\n      settled =\n        line >= currentChangedRangeEnd &&\n        canReuseCachedStates &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        if (nextRange[0] >= renderRangeEndLine) {\n          backgroundStartLine = nextRange[0];\n          backgroundChangedRangeIndex = changedRangeIndex;\n          break;\n        }\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          currentChangedRangeEnd = nextRange[1];\n          line++;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          currentChangedRangeEnd = nextRange[1];\n        }\n        settled = false;\n        continue;\n      }\n      line++;\n    }\n\n    if (canCacheTokenizedStates) {\n      if (line < renderRangeEndLine) {\n        this.#stateStack[line + 1] = state;\n      } else {\n        this.#stateStack[line] = state;\n      }\n    }\n\n    if (offscreenDirtyLines !== undefined && offscreenDirtyLines.size > 0) {\n      this.#onDeferTokenize(offscreenDirtyLines, this.#themeType);\n    }\n\n    if (backgroundStartLine !== undefined) {\n      this.#scheduleBackgroundTokenize(\n        backgroundStartLine,\n        changedLineRanges,\n        backgroundChangedRangeIndex\n      );\n    } else if (!settled && line < lineCount) {\n      const backgroundLine =\n        crossesRenderRangeEnd && dirtyStart >= viewStart\n          ? renderRangeEndLine\n          : dirtyStart < viewStart && !canReuseCachedStates\n            ? dirtyStart\n            : line;\n      this.#scheduleBackgroundTokenize(\n        backgroundLine,\n        canReuseCachedStates ? changedLineRanges : undefined,\n        changedRangeIndex\n      );\n    }\n\n    return dirtyLines;\n  }\n\n  prebuildStateStack(renderRange?: RenderRange): void {\n    this.#prebuildStateStack(renderRange);\n  }\n\n  stopBackgroundTokenize(): void {\n    if (this.#isStopped) {\n      return;\n    }\n    this.#isStopped = true;\n    this.#isPaused = false;\n    this.#lastLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#detachMessageListener();\n  }\n\n  pauseBackgroundTokenize(): void {\n    if (this.#isStopped || this.#isPaused) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization paused', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = true;\n  }\n\n  resumeBackgroundTokenize(): void {\n    if (\n      this.#isStopped ||\n      !this.#isPaused ||\n      this.#grammar === undefined ||\n      this.#lastLine < 0\n    ) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization resumed', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = false;\n    this.#postTokenizeMessage(this.#backgroundJobId);\n  }\n\n  #attachMessageListener(): void {\n    if (this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.addEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = true;\n  }\n\n  #detachMessageListener(): void {\n    if (!this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.removeEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = false;\n  }\n\n  #postTokenizeMessage(jobId: number): void {\n    // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay\n    globalThis.postMessage({ type: 'tokenize', jobId });\n  }\n\n  #scheduleBackgroundTokenize(\n    startLine: number,\n    changedLineRanges?: readonly [number, number][],\n    changedRangeIndex = 0\n  ): void {\n    const jobId = ++this.#backgroundJobId;\n\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization scheduled', {\n        jobId,\n        startLine,\n        changedLineRanges,\n        changedRangeIndex,\n      });\n    }\n\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = startLine;\n    this.#backgroundChangedLineRanges = changedLineRanges;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #tokenizeLineAt(\n    line: number,\n    state: StateStack\n  ): { resolvedTokens: Array<HighlightedToken>; state: StateStack } {\n    if (this.#grammar === undefined) {\n      throw new Error('Grammar not loaded');\n    }\n    const lineText = this.#textDocument.getLineText(line);\n    if (lineText.length > this.#tokenizeMaxLineLength) {\n      console.warn(\n        `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n      );\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    if (lineText === '' || lineText.trim() === '') {\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    const result = tokenizeLine(\n      this.#grammar,\n      this.#colorMap,\n      lineText,\n      state,\n      EditorTokenizer.TOKENIZE_TIME_LIMIT\n    );\n    return {\n      resolvedTokens: result.resolvedTokens,\n      state: result.ruleStack,\n    };\n  }\n\n  #buildStateStack(endAt: number) {\n    const boundedEndAt = Math.min(\n      Math.max(0, endAt),\n      this.#textDocument.lineCount\n    );\n    if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) {\n      return;\n    }\n    let line = this.#stateStack.length - 1;\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < boundedEndAt; line++) {\n      this.#stateStack[line] = state;\n      const lineText = this.#textDocument.getLineText(line);\n      if (\n        lineText.length <= this.#tokenizeMaxLineLength &&\n        lineText !== '' &&\n        lineText.trim() !== ''\n      ) {\n        state = this.#grammar.tokenizeLine2(\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        ).ruleStack;\n      }\n    }\n    this.#stateStack[line] = state;\n  }\n\n  #backgroundTokenize(jobId: number) {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar === undefined ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    const t = performance.now();\n    const lines = new Map<number, Array<HighlightedToken>>();\n    const totalLines = this.#textDocument.lineCount;\n    const changedLineRanges = this.#backgroundChangedLineRanges;\n\n    let line = this.#lastLine;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    let changedRangeIndex = this.#backgroundChangedRangeIndex;\n    let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];\n    for (; line < totalLines; ) {\n      this.#stateStack[line] = state;\n\n      const previousNextState =\n        currentChangedRangeEnd !== undefined\n          ? this.#stateStack[line + 1]\n          : undefined;\n      const lineText = this.#textDocument.getLineText(line);\n      if (lineText.length > this.#tokenizeMaxLineLength) {\n        console.warn(\n          `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n        );\n        lines.set(line, [[0, '', lineText]]);\n      } else if (lineText === '' || lineText.trim() === '') {\n        lines.set(line, [[0, '', lineText]]);\n      } else {\n        const ret = tokenizeLine(\n          this.#grammar,\n          this.#colorMap,\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        );\n        lines.set(line, ret.resolvedTokens);\n        state = ret.ruleStack;\n      }\n\n      this.#stateStack[line + 1] = state;\n      settled =\n        currentChangedRangeEnd !== undefined &&\n        line >= currentChangedRangeEnd &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      line++;\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges?.[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        currentChangedRangeEnd = nextRange[1];\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          settled = false;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          settled = false;\n          continue;\n        }\n      }\n\n      // limit the time of partial tokenize to 1ms\n      if (performance.now() - t > 1) {\n        break;\n      }\n    }\n\n    this.#onDeferTokenize(lines, this.#themeType);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n\n    if (settled || line >= totalLines) {\n      this.stopBackgroundTokenize();\n      return;\n    }\n\n    this.#lastLine = line;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#postTokenizeMessage(jobId);\n  }\n}\n\nexport function tokenizeLine(\n  grammar: IGrammar,\n  colorMap: string[],\n  lineText: string,\n  stateStack: StateStack,\n  timeLimit?: number\n): {\n  ruleStack: StateStack;\n  resolvedTokens: Array<HighlightedToken>;\n} {\n  const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);\n  if (result.stoppedEarly) {\n    console.warn(\n      `[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`\n    );\n  }\n  const rawTokens = result.tokens;\n  const tokensLength = rawTokens.length / 2;\n  const resolvedTokens: Array<HighlightedToken> = [];\n  for (let j = 0; j < tokensLength; j++) {\n    const offset = rawTokens[2 * j];\n    const nextOffset =\n      j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;\n    if (offset === nextOffset) {\n      // should never reach here, skip if happens anyway\n      continue;\n    }\n    const metadata = rawTokens[2 * j + 1];\n    const bg = EncodedTokenMetadata.getForeground(metadata);\n    const fg = colorMap[bg];\n    const tokenText = lineText.slice(offset, nextOffset);\n    resolvedTokens.push([offset, fg, tokenText]);\n  }\n  return {\n    ruleStack: result.ruleStack,\n    resolvedTokens,\n  };\n}\n\nexport function renderLineTokens(\n  tokens: Array<HighlightedToken>,\n  themeType: 'light' | 'dark'\n): (HTMLElement | string)[] {\n  return tokens.map(([char, fg, textContent]) => {\n    if (char === 0 && fg === '') {\n      if (textContent === '') {\n        return h('br');\n      }\n      return textContent;\n    }\n    return h('span', {\n      dataset: {\n        char: char.toString(),\n      },\n      style: `--diffs-token-${themeType}:${fg};`,\n      textContent: textContent,\n    });\n  });\n}\n"],"mappings":";;;;;;AA8BA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,sBAAsB;CAE7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,cAA4B,CAAC,QAAQ;CACrC,YAAoB;CACpB,aAAsB;CACtB,YAAqB;CACrB,mBAA2B;CAC3B;CACA,+BAAuC;CACvC,6BAAsC;CAEtC,sBAAsB,SAAS,OAAO,gBAA8B;EAClE,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,UAAU,KAAK,IACnB,eAAe,WAAW,WAAW,eAAe,YACpD,MAAKA,aAAc,UACpB;AACD,MAAI,MAAKC,YAAa,QAAW;AAC/B,SAAM,MAAKC,YAAa,aAAa,MAAKF,aAAc,WAAW;AACnE,SAAKC,UAAW,MAAKC,YAAa,YAChC,MAAKF,aAAc,WACpB;;AAEH,QAAKG,gBAAiB,QAAQ;IAC7B,IAAI;CAEP,cAAc,EAAE,WAAkC;AAChD,MAAI,OAAO,SAAS,YAAY,SAAS,KACvC;EAEF,MAAM,EAAE,MAAM,UAAU;AAIxB,MACE,SAAS,cACT,OAAO,UAAU,YACjB,UAAU,MAAKC,gBAEf,OAAKC,mBAAoB,MAAM;;CAInC,IAAI,YAA8B;AAChC,SAAO,MAAKC;;CAGd,YAAY,EACV,aACA,aACA,cACA,UACA,iBACA,WACuB;EACvB,MAAM,EACJ,YAAY,UACZ,QAAQ,gBACR,wBAAwB,QACtB;AACJ,QAAKC,iBAAkB,OAAO,WAAW,+BAA+B;AACxE,MAAI,cAAc,SAChB,OAAKD,YAAa,MAAKC,eAAgB,UAAU,SAAS;MAE1D,OAAKD,YAAa;AAEpB,MAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,WAAW,IAAI,kBAAkB,cAAc;AACnD,SAAK,MAAM,EAAE,MAAM,mBAAmB,UACpC,KACE,SAAS,gBACT,kBAAkB,SACjB,kBAAkB,WAAW,cAAc,WAAW,QAAQ,GAC/D;KACA,MAAME,cACJ,iBAAiB,SAAS,KAAK,CAAC,gBAAgB,SAC5C,SACA;AACN,WAAKC,gBAAiB,MAAMD,cAAYA,YAAU;AAClD;;KAGJ;AACF,YAAS,QAAQ,SAAS,iBAAiB,EAAE,YAAY,MAAM,CAAC;AAChE,YAAS,QAAQ,SAAS,MAAM,EAAE,YAAY,MAAM,CAAC;AACrD,SAAKE,WAAY,CACf,iBAAiB,MAAKH,gBAAiB,WAAW,MAAM;IACtD,MAAMC,cAAY,EAAE,UAAU,SAAS;AACvC,UAAKC,gBAAiB,MAAMD,cAAYA,YAAU;KAClD,QACI,SAAS,YAAY,CAC5B;;AAEH,QAAKN,cAAe;AACpB,QAAKF,eAAgB;AACrB,QAAKW,wBAAyB;AAC9B,QAAKC,WAAY;AACjB,QAAKC,kBAAmB;AACxB,QAAKC,QAAS,WAAW;AACzB,MAAI,YAAY,oBAAoB,CAAC,SAAS,aAAa,WAAW,CACpE,OAAKb,UAAW,YAAY,YAAY,aAAa,WAAW;AAElE,QAAKc,WAAY,EAAE;AACnB,QAAKC,SAAU,OAAO,UAAU,WAAW,QAAQ,MAAM,MAAKV,WAAY;;CAK5E,iBAAiB,WAAmB,WAA6B;AAC/D,QAAKA,YAAa;AAClB,QAAKU,SAAU,UAAU;AACzB,OAAK,wBAAwB;AAC7B,QAAKC,aAAc,CAAC,QAAQ;AAC5B,MAAI,MAAKhB,YAAa,UAAa,MAAKD,aAAc,YAAY,EAChE,OAAKkB,2BAA4B,EAAE;;CAIvC,UAAU,WAAmB;AAC3B,QAAKH,WAAY,MAAKb,YAAa,SAAS,UAAU,CAAC;EACvD,MAAM,EAAE,SAAS,EAAE,KAAK,MAAKA,YAAa,SAAS,UAAU;EAC7D,MAAM,sBAAsB,OAAO;EACnC,MAAM,0BAA0B,OAAO;EACvC,MAAM,mBAAmB,OAAO;EAChC,MAAM,yBAAyB,OAAO;EACtC,MAAM,mBAAmB,OAAO;EAChC,MAAM,sBAAsB,OAAO;EACnC,MAAM,+BACJ,OAAO;EACT,MAAM,iBAAiB,OAAO;EAC9B,MAAM,iBAAiB,OAAO;EAC9B,MAAM,oBAAoB,OAAO;EACjC,MAAM,kBAAkB,OAAO;AAC/B,QAAKU,SAAU;qCACkB,uBAAuB,uBAAuB;0CACzC,2BAA2B,uBAAuB;uCACrD,oBAAoB,yBAAyB;8CACtC,2BAA2B,wCAAwC;8CACnE,0BAA0B,mCAAmC;iCAC1E,uBAAuB,QAAQ;2CACrB,gCAAgC,QAAQ;kCACjD,oBAAoB,QAAQ;gCAC9B,kBAAkB,QAAQ;gCAC1B,kBAAkB,QAAQ;mCACvB,qBAAqB,QAAQ;iCAC/B,mBAAmB,QAAQ;OACrD;;CAGL,UAAgB;AACd,QAAKO,uBAAwB;AAC7B,OAAK,wBAAwB;AAC7B,QAAKT,UAAW,SAAS,YAAY,SAAS,CAAC;AAC/C,QAAKA,WAAY;;CAKnB,SACE,QACA,aACsC;AACtC,MAAI,MAAKT,YAAa,OACpB,OAAM,IAAI,MAAM,qBAAqB;EAGvC,MAAM,EAAE,cAAc,MAAKD;EAC3B,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,qBACJ,eAAe,WACX,YACA,KAAK,IAAI,eAAe,YAAY,UAAU;EAEpD,MAAM,aAAa,OAAO;EAC1B,MAAM,YAAY,KAAK,IAAI,cAAc,WAAW;EACpD,MAAM,wBACJ,gBAAgB,UAChB,eAAe,YACf,OAAO,YAAY,KACnB,aAAa,sBACb,OAAO,WAAW;EACpB,MAAM,uBAAuB,OAAO,cAAc;EAClD,MAAM,0BACJ,wBACA,gBAAgB,UAChB,cAAc;EAChB,MAAMoB,oBAAiD,uBAClD,OAAO,qBAAqB,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC,GAC3D,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC;EAClC,IAAI,mBAAmB;AACvB,MAAI,aAAa,WACf;QAAK,MAAM,CAAC,YAAY,aAAa,kBACnC,KAAI,aAAa,UACf,oBAAmB,KAAK,IACtB,kBACA,KAAK,IAAI,UAAU,YAAY,EAAE,CAClC;;EAIP,MAAM,4BACJ,oBAAoB,eACnB,wBAAwB,OAAO,YAAY;AAC9C,MAAI,qBACF,OAAKjB,gBAAiB,WAAW;OAC5B;AACL,SAAKc,WAAY,SAAS,KAAK,IAC7B,MAAKA,WAAY,QACjB,aAAa,EACd;AACD,OAAI,gBAAgB,UAAa,cAAc,UAC7C,OAAKd,gBAAiB,UAAU;;EAIpC,IAAI,oBAAoB;EACxB,IAAI,yBAAyB,kBAAkB,mBAAmB;EAClE,IAAIkB;EACJ,IAAI,8BAA8B;EAClC,IAAI,OAAO,uBACP,kBAAkB,mBAAmB,KACrC;EACJ,IAAI,QAAQ,MAAKJ,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,MAAMK,6BAAmD,IAAI,KAAK;EAClE,MAAMC,sBAEU,4CAA4B,IAAI,KAAK,GAAG;AACxD,MAAI,wBAAwB,UAAa,CAAC,sBAAsB;GAC9D,MAAM,eAAe,KAAK,IACxB,mBAAmB,GACnB,WACA,mBACD;AACD,OAAI,eAAe,YAAY;AAC7B,UAAKpB,gBAAiB,aAAa;IACnC,IAAI,gBAAgB;IACpB,IAAI,iBAAiB,MAAKc,WAAY,kBAAkB;AACxD,WAAO,gBAAgB,cAAc,iBAAiB;KACpD,MAAM,WAAW,MAAKO,eAAgB,eAAe,eAAe;AACpE,sBAAiB,SAAS;AAC1B,yBAAoB,IAAI,eAAe,SAAS,eAAe;;AAEjE,QAAI,wBACF,OAAKP,WAAY,gBAAgB;;;AAIvC,SAAO,OAAO,qBAAsB;GAClC,MAAM,oBAAoB,uBACtB,MAAKA,WAAY,OAAO,KACxB;AACJ,OAAI,wBACF,OAAKA,WAAY,QAAQ;GAG3B,MAAM,EAAE,gBAAgB,OAAO,cAAc,MAAKO,eAChD,MACA,MACD;AACD,WAAQ;AAER,OAAI,QAAQ,UACV,YAAW,IAAI,MAAM,eAAe;OAEpC,sBAAqB,IAAI,MAAM,eAAe;AAGhD,OAAI,wBACF,OAAKP,WAAY,OAAO,KAAK;AAE/B,aACE,QAAQ,0BACR,wBACA,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC,OAAI,SAAS;AACX;IACA,MAAM,YAAY,kBAAkB;AACpC,QAAI,cAAc,OAChB;AAEF,QAAI,UAAU,MAAM,oBAAoB;AACtC,2BAAsB,UAAU;AAChC,mCAA8B;AAC9B;;AAEF,QAAI,MAAKA,WAAY,UAAU,QAAQ,QAAW;AAChD,8BAAyB,UAAU;AACnC;WACK;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,8BAAyB,UAAU;;AAErC,cAAU;AACV;;AAEF;;AAGF,MAAI,wBACF,KAAI,OAAO,mBACT,OAAKA,WAAY,OAAO,KAAK;MAE7B,OAAKA,WAAY,QAAQ;AAI7B,MAAI,wBAAwB,UAAa,oBAAoB,OAAO,EAClE,OAAKJ,gBAAiB,qBAAqB,MAAKP,UAAW;AAG7D,MAAI,wBAAwB,OAC1B,OAAKY,2BACH,qBACA,mBACA,4BACD;WACQ,CAAC,WAAW,OAAO,WAAW;GACvC,MAAM,iBACJ,yBAAyB,cAAc,YACnC,qBACA,aAAa,aAAa,CAAC,uBACzB,aACA;AACR,SAAKA,2BACH,gBACA,uBAAuB,oBAAoB,QAC3C,kBACD;;AAGH,SAAO;;CAGT,mBAAmB,aAAiC;AAClD,QAAKO,mBAAoB,YAAY;;CAGvC,yBAA+B;AAC7B,MAAI,MAAKC,UACP;AAEF,QAAKA,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKX,uBAAwB;;CAG/B,0BAAgC;AAC9B,MAAI,MAAKO,aAAc,MAAKC,SAC1B;AAEF,MAAI,MAAKb,MACP,SAAQ,IAAI,iDAAiD,EAC3D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;;CAGnB,2BAAiC;AAC/B,MACE,MAAKD,aACL,CAAC,MAAKC,YACN,MAAK1B,YAAa,UAClB,MAAK2B,WAAY,EAEjB;AAEF,MAAI,MAAKd,MACP,SAAQ,IAAI,kDAAkD,EAC5D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;AACjB,QAAKI,oBAAqB,MAAK3B,gBAAiB;;CAGlD,yBAA+B;AAC7B,MAAI,MAAK4B,0BACP;AAEF,aAAW,iBAAiB,WAAW,MAAKC,UAAW;AACvD,QAAKD,4BAA6B;;CAGpC,yBAA+B;AAC7B,MAAI,CAAC,MAAKA,0BACR;AAEF,aAAW,oBAAoB,WAAW,MAAKC,UAAW;AAC1D,QAAKD,4BAA6B;;CAGpC,qBAAqB,OAAqB;AAExC,aAAW,YAAY;GAAE,MAAM;GAAY;GAAO,CAAC;;CAGrD,4BACE,WACA,mBACA,oBAAoB,GACd;EACN,MAAM,QAAQ,EAAE,MAAK5B;AAErB,MAAI,MAAKU,MACP,SAAQ,IAAI,oDAAoD;GAC9D;GACA;GACA;GACA;GACD,CAAC;AAGJ,QAAKY,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKI,uBAAwB;AAC7B,QAAKH,oBAAqB,MAAM;;CAGlC,gBACE,MACA,OACgE;AAChE,MAAI,MAAK9B,YAAa,OACpB,OAAM,IAAI,MAAM,qBAAqB;EAEvC,MAAM,WAAW,MAAKD,aAAc,YAAY,KAAK;AACrD,MAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,WAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC;IAAE;IAAO;;AAEvD,MAAI,aAAa,MAAM,SAAS,MAAM,KAAK,GACzC,QAAO;GAAE,gBAAgB,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC;GAAE;GAAO;EAEvD,MAAM,SAAS,aACb,MAAKV,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,SAAO;GACL,gBAAgB,OAAO;GACvB,OAAO,OAAO;GACf;;CAGH,iBAAiB,OAAe;EAC9B,MAAM,eAAe,KAAK,IACxB,KAAK,IAAI,GAAG,MAAM,EAClB,MAAKf,aAAc,UACpB;AACD,MAAI,MAAKiB,WAAY,SAAS,gBAAgB,MAAKhB,YAAa,OAC9D;EAEF,IAAI,OAAO,MAAKgB,WAAY,SAAS;EACrC,IAAI,QAAQ,MAAKA,WAAY,SAAS;AACtC,SAAO,OAAO,cAAc,QAAQ;AAClC,SAAKA,WAAY,QAAQ;GACzB,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OACE,SAAS,UAAU,MAAKW,yBACxB,aAAa,MACb,SAAS,MAAM,KAAK,GAEpB,SAAQ,MAAKV,QAAS,cACpB,UACA,OACA,gBAAgB,oBACjB,CAAC;;AAGN,QAAKgB,WAAY,QAAQ;;CAG3B,oBAAoB,OAAe;AACjC,MACE,MAAKS,aACL,MAAKC,YACL,MAAK1B,YAAa,UAClB,UAAU,MAAKG,gBAEf;EAGF,MAAM,IAAI,YAAY,KAAK;EAC3B,MAAM,wBAAQ,IAAI,KAAsC;EACxD,MAAM,aAAa,MAAKJ,aAAc;EACtC,MAAM,oBAAoB,MAAK6B;EAE/B,IAAI,OAAO,MAAKD;EAChB,IAAI,QAAQ,MAAKX,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,IAAI,oBAAoB,MAAKa;EAC7B,IAAI,yBAAyB,oBAAoB,qBAAqB;AACtE,SAAO,OAAO,aAAc;AAC1B,SAAKb,WAAY,QAAQ;GAEzB,MAAM,oBACJ,2BAA2B,SACvB,MAAKA,WAAY,OAAO,KACxB;GACN,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,YAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAM,IAAI,MAAM,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC,CAAC;cAC3B,aAAa,MAAM,SAAS,MAAM,KAAK,GAChD,OAAM,IAAI,MAAM,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC,CAAC;QAC/B;IACL,MAAM,MAAM,aACV,MAAKV,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,UAAM,IAAI,MAAM,IAAI,eAAe;AACnC,YAAQ,IAAI;;AAGd,SAAKE,WAAY,OAAO,KAAK;AAC7B,aACE,2BAA2B,UAC3B,QAAQ,0BACR,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC;AACA,OAAI,SAAS;AACX;IACA,MAAM,YAAY,oBAAoB;AACtC,QAAI,cAAc,OAChB;AAEF,6BAAyB,UAAU;AACnC,QAAI,MAAKA,WAAY,UAAU,QAAQ,OACrC,WAAU;SACL;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,eAAU;AACV;;;AAKJ,OAAI,YAAY,KAAK,GAAG,IAAI,EAC1B;;AAIJ,QAAKJ,gBAAiB,OAAO,MAAKP,UAAW;AAC7C,MAAI,MAAKoB,aAAc,MAAKC,YAAa,UAAU,MAAKvB,gBACtD;AAGF,MAAI,WAAW,QAAQ,YAAY;AACjC,QAAK,wBAAwB;AAC7B;;AAGF,QAAKwB,WAAY;AACjB,QAAKE,8BAA+B;AACpC,QAAKC,oBAAqB,MAAM;;;AAIpC,SAAgB,aACd,SACA,UACA,UACA,YACA,WAIA;CACA,MAAM,SAAS,QAAQ,cAAc,UAAU,YAAY,UAAU;AACrE,KAAI,OAAO,aACT,SAAQ,KACN,oDAAoD,SAAS,UAAU,GAAG,IAAI,GAC/E;CAEH,MAAM,YAAY,OAAO;CACzB,MAAM,eAAe,UAAU,SAAS;CACxC,MAAMI,iBAA0C,EAAE;AAClD,MAAK,IAAI,IAAI,GAAG,IAAI,cAAc,KAAK;EACrC,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,aACJ,IAAI,IAAI,eAAe,UAAU,IAAI,IAAI,KAAK,SAAS;AACzD,MAAI,WAAW,WAEb;EAEF,MAAM,WAAW,UAAU,IAAI,IAAI;EAEnC,MAAM,KAAK,SADA,qBAAqB,cAAc,SAAS;EAEvD,MAAM,YAAY,SAAS,MAAM,QAAQ,WAAW;AACpD,iBAAe,KAAK;GAAC;GAAQ;GAAI;GAAU,CAAC;;AAE9C,QAAO;EACL,WAAW,OAAO;EAClB;EACD;;AAGH,SAAgB,iBACd,QACA,WAC0B;AAC1B,QAAO,OAAO,KAAK,CAAC,MAAM,IAAI,iBAAiB;AAC7C,MAAI,SAAS,KAAK,OAAO,IAAI;AAC3B,OAAI,gBAAgB,GAClB,QAAO,EAAE,KAAK;AAEhB,UAAO;;AAET,SAAO,EAAE,QAAQ;GACf,SAAS,EACP,MAAM,KAAK,UAAU,EACtB;GACD,OAAO,iBAAiB,UAAU,GAAG,GAAG;GAC3B;GACd,CAAC;GACF"}
+{"version":3,"file":"tokenzier.js","names":["#textDocument","#grammar","#highlighter","#buildStateStack","#backgroundJobId","#backgroundTokenize","#themeType","#mediaQueryList","themeType","#emitThemeChange","#disposes","#tokenizeMaxLineLength","#setStyle","#onDeferTokenize","#debug","#colorMap","#setTheme","#stateStack","#scheduleBackgroundTokenize","#detachMessageListener","changedLineRanges: readonly [number, number][]","backgroundStartLine: number | undefined","dirtyLines: Map<number, Array<HighlightedToken>>","offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined","#tokenizeLineAt","#prebuildStateStack","#isStopped","#isPaused","#lastLine","#backgroundChangedLineRanges","#backgroundChangedRangeIndex","#postTokenizeMessage","#isMessageListenerAttached","#onMessage","#attachMessageListener","resolvedTokens: Array<HighlightedToken>"],"sources":["../../src/editor/tokenzier.ts"],"sourcesContent":["import {\n  EncodedTokenMetadata,\n  type IGrammar,\n  INITIAL,\n  type StateStack,\n} from 'shiki/textmate';\n\nimport { DEFAULT_THEMES } from '../constants';\nimport type {\n  BaseCodeOptions,\n  DiffsHighlighter,\n  HighlightedToken,\n  RenderRange,\n} from '../types';\nimport type { TextDocument, TextDocumentChange } from './textDocument';\nimport { addEventListener, debounce, h } from './utils';\n\nexport interface EditorTokenizerProps {\n  highlighter: DiffsHighlighter;\n  textDocument: TextDocument<unknown>;\n  codeOptions: BaseCodeOptions;\n  setStyle: (style: string) => void;\n  onDeferTokenize: (\n    lines: Map<number, Array<HighlightedToken>>,\n    themeType: 'dark' | 'light'\n  ) => void;\n  __debug?: boolean;\n}\n\n/** Stoppable code tokenizer for the editor */\nexport class EditorTokenizer {\n  static TOKENIZE_TIME_LIMIT = 500;\n\n  #highlighter: DiffsHighlighter;\n  #grammar: IGrammar | undefined;\n  #mediaQueryList: MediaQueryList;\n  #themeType: 'light' | 'dark';\n  #colorMap: string[];\n  #textDocument: TextDocument<unknown>;\n  #tokenizeMaxLineLength: number;\n  #setStyle: EditorTokenizerProps['setStyle'];\n  #onDeferTokenize: EditorTokenizerProps['onDeferTokenize'];\n  #debug: boolean;\n  #disposes?: (() => void)[];\n\n  // state\n  #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index\n  #lastLine: number = -1;\n  #isStopped: boolean = true;\n  #isPaused: boolean = false;\n  #backgroundJobId: number = 0;\n  #backgroundChangedLineRanges: readonly [number, number][] | undefined;\n  #backgroundChangedRangeIndex: number = 0;\n  #isMessageListenerAttached: boolean = false;\n\n  #prebuildStateStack = debounce(async (renderRange?: RenderRange) => {\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const endLine = Math.min(\n      totalLines === Infinity ? Infinity : startingLine + totalLines,\n      this.#textDocument.lineCount\n    );\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      await this.#highlighter.loadLanguage(this.#textDocument.languageId);\n      this.#grammar = this.#highlighter.getLanguage(\n        this.#textDocument.languageId\n      );\n    }\n    this.#buildStateStack(endLine);\n  }, 500);\n\n  #onMessage = ({ data }: MessageEvent<unknown>) => {\n    if (typeof data !== 'object' || data === null) {\n      return;\n    }\n    const { type, jobId } = data as {\n      type?: unknown;\n      jobId?: unknown;\n    };\n    if (\n      type === 'tokenize' &&\n      typeof jobId === 'number' &&\n      jobId === this.#backgroundJobId\n    ) {\n      this.#backgroundTokenize(jobId);\n    }\n  };\n\n  get themeType(): 'light' | 'dark' {\n    return this.#themeType;\n  }\n\n  constructor({\n    codeOptions,\n    highlighter,\n    textDocument,\n    setStyle,\n    onDeferTokenize,\n    __debug,\n  }: EditorTokenizerProps) {\n    const {\n      themeType = 'system',\n      theme = DEFAULT_THEMES,\n      tokenizeMaxLineLength = 1000,\n    } = codeOptions;\n    this.#mediaQueryList = window.matchMedia('(prefers-color-scheme: dark)');\n    if (themeType === 'system') {\n      this.#themeType = this.#mediaQueryList.matches ? 'dark' : 'light';\n    } else {\n      this.#themeType = themeType;\n    }\n    if (typeof theme !== 'string') {\n      const observer = new MutationObserver((mutations) => {\n        for (const { type, attributeName } of mutations) {\n          if (\n            type === 'attributes' &&\n            attributeName !== null &&\n            (attributeName === 'class' || attributeName.startsWith('data-'))\n          ) {\n            const themeType =\n              getComputedStyle(document.body).colorScheme === 'dark'\n                ? 'dark'\n                : 'light';\n            this.#emitThemeChange(theme[themeType], themeType);\n            break;\n          }\n        }\n      });\n      observer.observe(document.documentElement, { attributes: true });\n      observer.observe(document.body, { attributes: true });\n      this.#disposes = [\n        addEventListener(this.#mediaQueryList, 'change', (e) => {\n          const themeType = e.matches ? 'dark' : 'light';\n          this.#emitThemeChange(theme[themeType], themeType);\n        }),\n        () => observer.disconnect(),\n      ];\n    }\n    this.#highlighter = highlighter;\n    this.#textDocument = textDocument;\n    this.#tokenizeMaxLineLength = tokenizeMaxLineLength;\n    this.#setStyle = setStyle;\n    this.#onDeferTokenize = onDeferTokenize;\n    this.#debug = __debug ?? false;\n    if (\n      !isGrammarlessLanguage(textDocument.languageId) &&\n      highlighter.getLoadedLanguages().includes(textDocument.languageId)\n    ) {\n      this.#grammar = highlighter.getLanguage(textDocument.languageId);\n    }\n    this.#colorMap = [];\n    this.#setTheme(typeof theme === 'string' ? theme : theme[this.#themeType]);\n  }\n\n  // By default, diffs components support dual themes, but the tokenizer only renders\n  // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document.\n  #emitThemeChange(themeName: string, themeType: 'light' | 'dark') {\n    this.#themeType = themeType;\n    this.#setTheme(themeName);\n    this.stopBackgroundTokenize();\n    this.#stateStack = [INITIAL];\n    if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) {\n      this.#scheduleBackgroundTokenize(0);\n    }\n  }\n\n  #setTheme(themeName: string) {\n    this.#colorMap = this.#highlighter.setTheme(themeName).colorMap;\n    const { colors = {} } = this.#highlighter.getTheme(themeName);\n    const selectionBackground = colors['editor.selectionBackground'];\n    const lineHighlightBackground = colors['editor.lineHighlightBackground'];\n    const gutterForeground = colors['editorLineNumber.foreground'];\n    const gutterActiveForeground = colors['editorLineNumber.activeForeground'];\n    const cursorForeground = colors['editorCursor.foreground'];\n    const findMatchBackground = colors['editor.findMatchBackground'];\n    const findMatchHighlightBackground =\n      colors['editor.findMatchHighlightBackground'];\n    const hintForeground = colors['editorHint.foreground'];\n    const infoForeground = colors['editorInfo.foreground'];\n    const warningForeground = colors['editorWarning.foreground'];\n    const errorForeground = colors['editorError.foreground'];\n    this.#setStyle(`:host {\n      --diffs-editor-selection-bg: ${selectionBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-highlight-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg)'};\n      --diffs-editor-line-number-fg: ${gutterForeground ?? 'var(--diffs-fg-number)'};\n      --diffs-editor-line-number-active-bg: ${lineHighlightBackground ?? 'var(--diffs-line-bg, var(--diffs-bg))'};\n      --diffs-editor-line-number-active-fg: ${gutterActiveForeground ?? 'var(--diffs-selection-number-fg)'};\n      --diffs-editor-match-bg: ${findMatchBackground ?? 'unset'};\n      --diffs-editor-match-highlight-bg: ${findMatchHighlightBackground ?? 'unset'};\n      --diffs-editor-cursor-fg: ${cursorForeground ?? 'unset'};\n      --diffs-editor-hint-fg: ${hintForeground ?? 'unset'};\n      --diffs-editor-info-fg: ${infoForeground ?? 'unset'};\n      --diffs-editor-warning-fg: ${warningForeground ?? 'unset'};\n      --diffs-editor-error-fg: ${errorForeground ?? 'unset'};\n    }`);\n  }\n\n  cleanUp(): void {\n    this.stopBackgroundTokenize();\n    this.#detachMessageListener();\n    this.#disposes?.forEach((dispose) => dispose());\n    this.#disposes = undefined;\n  }\n\n  // to use `tokenize`, call `prebuildStateStackMap` first to prebuild\n  // the state stack map for the given render range.\n  tokenize(\n    change: TextDocumentChange,\n    renderRange?: RenderRange\n  ): Map<number, Array<HighlightedToken>> {\n    if (\n      this.#grammar === undefined &&\n      !isGrammarlessLanguage(this.#textDocument.languageId)\n    ) {\n      throw new Error('Grammar not loaded');\n    }\n\n    const { lineCount } = this.#textDocument;\n    const { startingLine = 0, totalLines = Infinity } = renderRange ?? {};\n    const renderRangeEndLine =\n      totalLines === Infinity\n        ? lineCount\n        : Math.min(startingLine + totalLines, lineCount);\n\n    const dirtyStart = change.startLine;\n    const viewStart = Math.max(startingLine, dirtyStart);\n    const crossesRenderRangeEnd =\n      renderRange !== undefined &&\n      totalLines !== Infinity &&\n      change.lineDelta > 0 &&\n      dirtyStart < renderRangeEndLine &&\n      change.endLine >= renderRangeEndLine;\n    const canReuseCachedStates = change.lineDelta === 0;\n    const canCacheTokenizedStates =\n      canReuseCachedStates ||\n      renderRange === undefined ||\n      dirtyStart >= viewStart;\n    const changedLineRanges: readonly [number, number][] = canReuseCachedStates\n      ? (change.changedLineRanges ?? [[dirtyStart, change.endLine]])\n      : [[dirtyStart, change.endLine]];\n    let offscreenSyncEnd = -1;\n    if (dirtyStart < viewStart) {\n      for (const [rangeStart, rangeEnd] of changedLineRanges) {\n        if (rangeStart < viewStart) {\n          offscreenSyncEnd = Math.max(\n            offscreenSyncEnd,\n            Math.min(rangeEnd, viewStart - 1)\n          );\n        }\n      }\n    }\n    const shouldFlushOffscreenLines =\n      offscreenSyncEnd >= dirtyStart &&\n      (canReuseCachedStates || change.lineDelta < 0);\n    if (canReuseCachedStates) {\n      this.#buildStateStack(dirtyStart);\n    } else {\n      this.#stateStack.length = Math.min(\n        this.#stateStack.length,\n        dirtyStart + 1\n      );\n      if (renderRange === undefined || dirtyStart >= viewStart) {\n        this.#buildStateStack(viewStart);\n      }\n    }\n\n    let changedRangeIndex = 0;\n    let currentChangedRangeEnd = changedLineRanges[changedRangeIndex][1];\n    let backgroundStartLine: number | undefined;\n    let backgroundChangedRangeIndex = 0;\n    let line = canReuseCachedStates\n      ? changedLineRanges[changedRangeIndex][0]\n      : viewStart;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    const dirtyLines: Map<number, Array<HighlightedToken>> = new Map();\n    const offscreenDirtyLines:\n      | Map<number, Array<HighlightedToken>>\n      | undefined = shouldFlushOffscreenLines ? new Map() : undefined;\n    if (offscreenDirtyLines !== undefined && !canReuseCachedStates) {\n      const offscreenEnd = Math.min(\n        offscreenSyncEnd + 1,\n        viewStart,\n        renderRangeEndLine\n      );\n      if (offscreenEnd > dirtyStart) {\n        this.#buildStateStack(offscreenEnd);\n        let offscreenLine = dirtyStart;\n        let offscreenState = this.#stateStack[offscreenLine] ?? INITIAL;\n        for (; offscreenLine < offscreenEnd; offscreenLine++) {\n          const resolved = this.#tokenizeLineAt(offscreenLine, offscreenState);\n          offscreenState = resolved.state;\n          offscreenDirtyLines.set(offscreenLine, resolved.resolvedTokens);\n        }\n        if (canCacheTokenizedStates) {\n          this.#stateStack[offscreenEnd] = offscreenState;\n        }\n      }\n    }\n    for (; line < renderRangeEndLine; ) {\n      const previousNextState = canReuseCachedStates\n        ? this.#stateStack[line + 1]\n        : undefined;\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line] = state;\n      }\n\n      const { resolvedTokens, state: nextState } = this.#tokenizeLineAt(\n        line,\n        state\n      );\n      state = nextState;\n\n      if (line >= viewStart) {\n        dirtyLines.set(line, resolvedTokens);\n      } else {\n        offscreenDirtyLines?.set(line, resolvedTokens);\n      }\n\n      if (canCacheTokenizedStates) {\n        this.#stateStack[line + 1] = state;\n      }\n      settled =\n        line >= currentChangedRangeEnd &&\n        canReuseCachedStates &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        if (nextRange[0] >= renderRangeEndLine) {\n          backgroundStartLine = nextRange[0];\n          backgroundChangedRangeIndex = changedRangeIndex;\n          break;\n        }\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          currentChangedRangeEnd = nextRange[1];\n          line++;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          currentChangedRangeEnd = nextRange[1];\n        }\n        settled = false;\n        continue;\n      }\n      line++;\n    }\n\n    if (canCacheTokenizedStates) {\n      if (line < renderRangeEndLine) {\n        this.#stateStack[line + 1] = state;\n      } else {\n        this.#stateStack[line] = state;\n      }\n    }\n\n    if (offscreenDirtyLines !== undefined && offscreenDirtyLines.size > 0) {\n      this.#onDeferTokenize(offscreenDirtyLines, this.#themeType);\n    }\n\n    if (backgroundStartLine !== undefined) {\n      this.#scheduleBackgroundTokenize(\n        backgroundStartLine,\n        changedLineRanges,\n        backgroundChangedRangeIndex\n      );\n    } else if (!settled && line < lineCount) {\n      const backgroundLine =\n        crossesRenderRangeEnd && dirtyStart >= viewStart\n          ? renderRangeEndLine\n          : dirtyStart < viewStart && !canReuseCachedStates\n            ? dirtyStart\n            : line;\n      this.#scheduleBackgroundTokenize(\n        backgroundLine,\n        canReuseCachedStates ? changedLineRanges : undefined,\n        changedRangeIndex\n      );\n    }\n\n    return dirtyLines;\n  }\n\n  prebuildStateStack(renderRange?: RenderRange): void {\n    this.#prebuildStateStack(renderRange);\n  }\n\n  stopBackgroundTokenize(): void {\n    if (this.#isStopped) {\n      return;\n    }\n    this.#isStopped = true;\n    this.#isPaused = false;\n    this.#lastLine = -1;\n    this.#backgroundChangedLineRanges = undefined;\n    this.#backgroundChangedRangeIndex = 0;\n    this.#detachMessageListener();\n  }\n\n  pauseBackgroundTokenize(): void {\n    if (this.#isStopped || this.#isPaused) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization paused', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = true;\n  }\n\n  resumeBackgroundTokenize(): void {\n    if (\n      this.#isStopped ||\n      !this.#isPaused ||\n      this.#grammar === undefined ||\n      this.#lastLine < 0\n    ) {\n      return;\n    }\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization resumed', {\n        jobId: this.#backgroundJobId,\n      });\n    }\n    this.#isPaused = false;\n    this.#postTokenizeMessage(this.#backgroundJobId);\n  }\n\n  #attachMessageListener(): void {\n    if (this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.addEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = true;\n  }\n\n  #detachMessageListener(): void {\n    if (!this.#isMessageListenerAttached) {\n      return;\n    }\n    globalThis.removeEventListener('message', this.#onMessage);\n    this.#isMessageListenerAttached = false;\n  }\n\n  #postTokenizeMessage(jobId: number): void {\n    // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay\n    globalThis.postMessage({ type: 'tokenize', jobId });\n  }\n\n  #scheduleBackgroundTokenize(\n    startLine: number,\n    changedLineRanges?: readonly [number, number][],\n    changedRangeIndex = 0\n  ): void {\n    if (isGrammarlessLanguage(this.#textDocument.languageId)) {\n      return;\n    }\n\n    const jobId = ++this.#backgroundJobId;\n\n    if (this.#debug) {\n      console.log('[diffs/editor] background tokenization scheduled', {\n        jobId,\n        startLine,\n        changedLineRanges,\n        changedRangeIndex,\n      });\n    }\n\n    this.#isStopped = false;\n    this.#isPaused = false;\n    this.#lastLine = startLine;\n    this.#backgroundChangedLineRanges = changedLineRanges;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#attachMessageListener();\n    this.#postTokenizeMessage(jobId);\n  }\n\n  #tokenizeLineAt(\n    line: number,\n    state: StateStack\n  ): { resolvedTokens: Array<HighlightedToken>; state: StateStack } {\n    const lineText = this.#textDocument.getLineText(line);\n    if (lineText.length > this.#tokenizeMaxLineLength) {\n      console.warn(\n        `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n      );\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    if (\n      this.#grammar === undefined ||\n      lineText === '' ||\n      lineText.trim() === ''\n    ) {\n      return { resolvedTokens: [[0, '', lineText]], state };\n    }\n    const result = tokenizeLine(\n      this.#grammar,\n      this.#colorMap,\n      lineText,\n      state,\n      EditorTokenizer.TOKENIZE_TIME_LIMIT\n    );\n    return {\n      resolvedTokens: result.resolvedTokens,\n      state: result.ruleStack,\n    };\n  }\n\n  #buildStateStack(endAt: number) {\n    const boundedEndAt = Math.min(\n      Math.max(0, endAt),\n      this.#textDocument.lineCount\n    );\n    if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) {\n      return;\n    }\n    let line = this.#stateStack.length - 1;\n    let state = this.#stateStack[line] ?? INITIAL;\n    for (; line < boundedEndAt; line++) {\n      this.#stateStack[line] = state;\n      const lineText = this.#textDocument.getLineText(line);\n      if (\n        lineText.length <= this.#tokenizeMaxLineLength &&\n        lineText !== '' &&\n        lineText.trim() !== ''\n      ) {\n        state = this.#grammar.tokenizeLine2(\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        ).ruleStack;\n      }\n    }\n    this.#stateStack[line] = state;\n  }\n\n  #backgroundTokenize(jobId: number) {\n    if (\n      this.#isStopped ||\n      this.#isPaused ||\n      this.#grammar === undefined ||\n      jobId !== this.#backgroundJobId\n    ) {\n      return;\n    }\n\n    const t = performance.now();\n    const lines = new Map<number, Array<HighlightedToken>>();\n    const totalLines = this.#textDocument.lineCount;\n    const changedLineRanges = this.#backgroundChangedLineRanges;\n\n    let line = this.#lastLine;\n    let state = this.#stateStack[line] ?? INITIAL;\n    let settled = false;\n    let changedRangeIndex = this.#backgroundChangedRangeIndex;\n    let currentChangedRangeEnd = changedLineRanges?.[changedRangeIndex]?.[1];\n    for (; line < totalLines; ) {\n      this.#stateStack[line] = state;\n\n      const previousNextState =\n        currentChangedRangeEnd !== undefined\n          ? this.#stateStack[line + 1]\n          : undefined;\n      const lineText = this.#textDocument.getLineText(line);\n      if (lineText.length > this.#tokenizeMaxLineLength) {\n        console.warn(\n          `[diffs] Line(${line}) too long to tokenize: ${lineText.length}`\n        );\n        lines.set(line, [[0, '', lineText]]);\n      } else if (lineText === '' || lineText.trim() === '') {\n        lines.set(line, [[0, '', lineText]]);\n      } else {\n        const ret = tokenizeLine(\n          this.#grammar,\n          this.#colorMap,\n          lineText,\n          state,\n          EditorTokenizer.TOKENIZE_TIME_LIMIT\n        );\n        lines.set(line, ret.resolvedTokens);\n        state = ret.ruleStack;\n      }\n\n      this.#stateStack[line + 1] = state;\n      settled =\n        currentChangedRangeEnd !== undefined &&\n        line >= currentChangedRangeEnd &&\n        previousNextState !== undefined &&\n        state.equals(previousNextState);\n      line++;\n      if (settled) {\n        changedRangeIndex++;\n        const nextRange = changedLineRanges?.[changedRangeIndex];\n        if (nextRange === undefined) {\n          break;\n        }\n        currentChangedRangeEnd = nextRange[1];\n        if (this.#stateStack[nextRange[0]] === undefined) {\n          settled = false;\n        } else {\n          line = nextRange[0];\n          state = this.#stateStack[line] ?? state;\n          settled = false;\n          continue;\n        }\n      }\n\n      // limit the time of partial tokenize to 1ms\n      if (performance.now() - t > 1) {\n        break;\n      }\n    }\n\n    this.#onDeferTokenize(lines, this.#themeType);\n    if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) {\n      return;\n    }\n\n    if (settled || line >= totalLines) {\n      this.stopBackgroundTokenize();\n      return;\n    }\n\n    this.#lastLine = line;\n    this.#backgroundChangedRangeIndex = changedRangeIndex;\n    this.#postTokenizeMessage(jobId);\n  }\n}\n\nexport function tokenizeLine(\n  grammar: IGrammar,\n  colorMap: string[],\n  lineText: string,\n  stateStack: StateStack,\n  timeLimit?: number\n): {\n  ruleStack: StateStack;\n  resolvedTokens: Array<HighlightedToken>;\n} {\n  const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);\n  if (result.stoppedEarly) {\n    console.warn(\n      `[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`\n    );\n  }\n  const rawTokens = result.tokens;\n  const tokensLength = rawTokens.length / 2;\n  const resolvedTokens: Array<HighlightedToken> = [];\n  for (let j = 0; j < tokensLength; j++) {\n    const offset = rawTokens[2 * j];\n    const nextOffset =\n      j + 1 < tokensLength ? rawTokens[2 * j + 2] : lineText.length;\n    if (offset === nextOffset) {\n      // should never reach here, skip if happens anyway\n      continue;\n    }\n    const metadata = rawTokens[2 * j + 1];\n    const bg = EncodedTokenMetadata.getForeground(metadata);\n    const fg = colorMap[bg];\n    const tokenText = lineText.slice(offset, nextOffset);\n    resolvedTokens.push([offset, fg, tokenText]);\n  }\n  return {\n    ruleStack: result.ruleStack,\n    resolvedTokens,\n  };\n}\n\nexport function renderLineTokens(\n  tokens: Array<HighlightedToken>,\n  themeType: 'light' | 'dark'\n): (HTMLElement | string)[] {\n  return tokens.map(([char, fg, textContent]) => {\n    if (char === 0 && fg === '') {\n      if (textContent === '') {\n        return h('br');\n      }\n      return textContent;\n    }\n    return h('span', {\n      dataset: {\n        char: char.toString(),\n      },\n      style: `--diffs-token-${themeType}:${fg};`,\n      textContent: textContent,\n    });\n  });\n}\n\n// Shiki special-cases `text` and `ansi` in codeToHast but does not expose grammars.\nfunction isGrammarlessLanguage(languageId: string): boolean {\n  return languageId === 'text' || languageId === 'ansi';\n}\n"],"mappings":";;;;;;AA8BA,IAAa,kBAAb,MAAa,gBAAgB;CAC3B,OAAO,sBAAsB;CAE7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,cAA4B,CAAC,QAAQ;CACrC,YAAoB;CACpB,aAAsB;CACtB,YAAqB;CACrB,mBAA2B;CAC3B;CACA,+BAAuC;CACvC,6BAAsC;CAEtC,sBAAsB,SAAS,OAAO,gBAA8B;EAClE,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,UAAU,KAAK,IACnB,eAAe,WAAW,WAAW,eAAe,YACpD,MAAKA,aAAc,UACpB;AACD,MACE,MAAKC,YAAa,UAClB,CAAC,sBAAsB,MAAKD,aAAc,WAAW,EACrD;AACA,SAAM,MAAKE,YAAa,aAAa,MAAKF,aAAc,WAAW;AACnE,SAAKC,UAAW,MAAKC,YAAa,YAChC,MAAKF,aAAc,WACpB;;AAEH,QAAKG,gBAAiB,QAAQ;IAC7B,IAAI;CAEP,cAAc,EAAE,WAAkC;AAChD,MAAI,OAAO,SAAS,YAAY,SAAS,KACvC;EAEF,MAAM,EAAE,MAAM,UAAU;AAIxB,MACE,SAAS,cACT,OAAO,UAAU,YACjB,UAAU,MAAKC,gBAEf,OAAKC,mBAAoB,MAAM;;CAInC,IAAI,YAA8B;AAChC,SAAO,MAAKC;;CAGd,YAAY,EACV,aACA,aACA,cACA,UACA,iBACA,WACuB;EACvB,MAAM,EACJ,YAAY,UACZ,QAAQ,gBACR,wBAAwB,QACtB;AACJ,QAAKC,iBAAkB,OAAO,WAAW,+BAA+B;AACxE,MAAI,cAAc,SAChB,OAAKD,YAAa,MAAKC,eAAgB,UAAU,SAAS;MAE1D,OAAKD,YAAa;AAEpB,MAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,WAAW,IAAI,kBAAkB,cAAc;AACnD,SAAK,MAAM,EAAE,MAAM,mBAAmB,UACpC,KACE,SAAS,gBACT,kBAAkB,SACjB,kBAAkB,WAAW,cAAc,WAAW,QAAQ,GAC/D;KACA,MAAME,cACJ,iBAAiB,SAAS,KAAK,CAAC,gBAAgB,SAC5C,SACA;AACN,WAAKC,gBAAiB,MAAMD,cAAYA,YAAU;AAClD;;KAGJ;AACF,YAAS,QAAQ,SAAS,iBAAiB,EAAE,YAAY,MAAM,CAAC;AAChE,YAAS,QAAQ,SAAS,MAAM,EAAE,YAAY,MAAM,CAAC;AACrD,SAAKE,WAAY,CACf,iBAAiB,MAAKH,gBAAiB,WAAW,MAAM;IACtD,MAAMC,cAAY,EAAE,UAAU,SAAS;AACvC,UAAKC,gBAAiB,MAAMD,cAAYA,YAAU;KAClD,QACI,SAAS,YAAY,CAC5B;;AAEH,QAAKN,cAAe;AACpB,QAAKF,eAAgB;AACrB,QAAKW,wBAAyB;AAC9B,QAAKC,WAAY;AACjB,QAAKC,kBAAmB;AACxB,QAAKC,QAAS,WAAW;AACzB,MACE,CAAC,sBAAsB,aAAa,WAAW,IAC/C,YAAY,oBAAoB,CAAC,SAAS,aAAa,WAAW,CAElE,OAAKb,UAAW,YAAY,YAAY,aAAa,WAAW;AAElE,QAAKc,WAAY,EAAE;AACnB,QAAKC,SAAU,OAAO,UAAU,WAAW,QAAQ,MAAM,MAAKV,WAAY;;CAK5E,iBAAiB,WAAmB,WAA6B;AAC/D,QAAKA,YAAa;AAClB,QAAKU,SAAU,UAAU;AACzB,OAAK,wBAAwB;AAC7B,QAAKC,aAAc,CAAC,QAAQ;AAC5B,MAAI,MAAKhB,YAAa,UAAa,MAAKD,aAAc,YAAY,EAChE,OAAKkB,2BAA4B,EAAE;;CAIvC,UAAU,WAAmB;AAC3B,QAAKH,WAAY,MAAKb,YAAa,SAAS,UAAU,CAAC;EACvD,MAAM,EAAE,SAAS,EAAE,KAAK,MAAKA,YAAa,SAAS,UAAU;EAC7D,MAAM,sBAAsB,OAAO;EACnC,MAAM,0BAA0B,OAAO;EACvC,MAAM,mBAAmB,OAAO;EAChC,MAAM,yBAAyB,OAAO;EACtC,MAAM,mBAAmB,OAAO;EAChC,MAAM,sBAAsB,OAAO;EACnC,MAAM,+BACJ,OAAO;EACT,MAAM,iBAAiB,OAAO;EAC9B,MAAM,iBAAiB,OAAO;EAC9B,MAAM,oBAAoB,OAAO;EACjC,MAAM,kBAAkB,OAAO;AAC/B,QAAKU,SAAU;qCACkB,uBAAuB,uBAAuB;0CACzC,2BAA2B,uBAAuB;uCACrD,oBAAoB,yBAAyB;8CACtC,2BAA2B,wCAAwC;8CACnE,0BAA0B,mCAAmC;iCAC1E,uBAAuB,QAAQ;2CACrB,gCAAgC,QAAQ;kCACjD,oBAAoB,QAAQ;gCAC9B,kBAAkB,QAAQ;gCAC1B,kBAAkB,QAAQ;mCACvB,qBAAqB,QAAQ;iCAC/B,mBAAmB,QAAQ;OACrD;;CAGL,UAAgB;AACd,OAAK,wBAAwB;AAC7B,QAAKO,uBAAwB;AAC7B,QAAKT,UAAW,SAAS,YAAY,SAAS,CAAC;AAC/C,QAAKA,WAAY;;CAKnB,SACE,QACA,aACsC;AACtC,MACE,MAAKT,YAAa,UAClB,CAAC,sBAAsB,MAAKD,aAAc,WAAW,CAErD,OAAM,IAAI,MAAM,qBAAqB;EAGvC,MAAM,EAAE,cAAc,MAAKA;EAC3B,MAAM,EAAE,eAAe,GAAG,aAAa,aAAa,eAAe,EAAE;EACrE,MAAM,qBACJ,eAAe,WACX,YACA,KAAK,IAAI,eAAe,YAAY,UAAU;EAEpD,MAAM,aAAa,OAAO;EAC1B,MAAM,YAAY,KAAK,IAAI,cAAc,WAAW;EACpD,MAAM,wBACJ,gBAAgB,UAChB,eAAe,YACf,OAAO,YAAY,KACnB,aAAa,sBACb,OAAO,WAAW;EACpB,MAAM,uBAAuB,OAAO,cAAc;EAClD,MAAM,0BACJ,wBACA,gBAAgB,UAChB,cAAc;EAChB,MAAMoB,oBAAiD,uBAClD,OAAO,qBAAqB,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC,GAC3D,CAAC,CAAC,YAAY,OAAO,QAAQ,CAAC;EAClC,IAAI,mBAAmB;AACvB,MAAI,aAAa,WACf;QAAK,MAAM,CAAC,YAAY,aAAa,kBACnC,KAAI,aAAa,UACf,oBAAmB,KAAK,IACtB,kBACA,KAAK,IAAI,UAAU,YAAY,EAAE,CAClC;;EAIP,MAAM,4BACJ,oBAAoB,eACnB,wBAAwB,OAAO,YAAY;AAC9C,MAAI,qBACF,OAAKjB,gBAAiB,WAAW;OAC5B;AACL,SAAKc,WAAY,SAAS,KAAK,IAC7B,MAAKA,WAAY,QACjB,aAAa,EACd;AACD,OAAI,gBAAgB,UAAa,cAAc,UAC7C,OAAKd,gBAAiB,UAAU;;EAIpC,IAAI,oBAAoB;EACxB,IAAI,yBAAyB,kBAAkB,mBAAmB;EAClE,IAAIkB;EACJ,IAAI,8BAA8B;EAClC,IAAI,OAAO,uBACP,kBAAkB,mBAAmB,KACrC;EACJ,IAAI,QAAQ,MAAKJ,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,MAAMK,6BAAmD,IAAI,KAAK;EAClE,MAAMC,sBAEU,4CAA4B,IAAI,KAAK,GAAG;AACxD,MAAI,wBAAwB,UAAa,CAAC,sBAAsB;GAC9D,MAAM,eAAe,KAAK,IACxB,mBAAmB,GACnB,WACA,mBACD;AACD,OAAI,eAAe,YAAY;AAC7B,UAAKpB,gBAAiB,aAAa;IACnC,IAAI,gBAAgB;IACpB,IAAI,iBAAiB,MAAKc,WAAY,kBAAkB;AACxD,WAAO,gBAAgB,cAAc,iBAAiB;KACpD,MAAM,WAAW,MAAKO,eAAgB,eAAe,eAAe;AACpE,sBAAiB,SAAS;AAC1B,yBAAoB,IAAI,eAAe,SAAS,eAAe;;AAEjE,QAAI,wBACF,OAAKP,WAAY,gBAAgB;;;AAIvC,SAAO,OAAO,qBAAsB;GAClC,MAAM,oBAAoB,uBACtB,MAAKA,WAAY,OAAO,KACxB;AACJ,OAAI,wBACF,OAAKA,WAAY,QAAQ;GAG3B,MAAM,EAAE,gBAAgB,OAAO,cAAc,MAAKO,eAChD,MACA,MACD;AACD,WAAQ;AAER,OAAI,QAAQ,UACV,YAAW,IAAI,MAAM,eAAe;OAEpC,sBAAqB,IAAI,MAAM,eAAe;AAGhD,OAAI,wBACF,OAAKP,WAAY,OAAO,KAAK;AAE/B,aACE,QAAQ,0BACR,wBACA,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC,OAAI,SAAS;AACX;IACA,MAAM,YAAY,kBAAkB;AACpC,QAAI,cAAc,OAChB;AAEF,QAAI,UAAU,MAAM,oBAAoB;AACtC,2BAAsB,UAAU;AAChC,mCAA8B;AAC9B;;AAEF,QAAI,MAAKA,WAAY,UAAU,QAAQ,QAAW;AAChD,8BAAyB,UAAU;AACnC;WACK;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,8BAAyB,UAAU;;AAErC,cAAU;AACV;;AAEF;;AAGF,MAAI,wBACF,KAAI,OAAO,mBACT,OAAKA,WAAY,OAAO,KAAK;MAE7B,OAAKA,WAAY,QAAQ;AAI7B,MAAI,wBAAwB,UAAa,oBAAoB,OAAO,EAClE,OAAKJ,gBAAiB,qBAAqB,MAAKP,UAAW;AAG7D,MAAI,wBAAwB,OAC1B,OAAKY,2BACH,qBACA,mBACA,4BACD;WACQ,CAAC,WAAW,OAAO,WAAW;GACvC,MAAM,iBACJ,yBAAyB,cAAc,YACnC,qBACA,aAAa,aAAa,CAAC,uBACzB,aACA;AACR,SAAKA,2BACH,gBACA,uBAAuB,oBAAoB,QAC3C,kBACD;;AAGH,SAAO;;CAGT,mBAAmB,aAAiC;AAClD,QAAKO,mBAAoB,YAAY;;CAGvC,yBAA+B;AAC7B,MAAI,MAAKC,UACP;AAEF,QAAKA,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKX,uBAAwB;;CAG/B,0BAAgC;AAC9B,MAAI,MAAKO,aAAc,MAAKC,SAC1B;AAEF,MAAI,MAAKb,MACP,SAAQ,IAAI,iDAAiD,EAC3D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;;CAGnB,2BAAiC;AAC/B,MACE,MAAKD,aACL,CAAC,MAAKC,YACN,MAAK1B,YAAa,UAClB,MAAK2B,WAAY,EAEjB;AAEF,MAAI,MAAKd,MACP,SAAQ,IAAI,kDAAkD,EAC5D,OAAO,MAAKV,iBACb,CAAC;AAEJ,QAAKuB,WAAY;AACjB,QAAKI,oBAAqB,MAAK3B,gBAAiB;;CAGlD,yBAA+B;AAC7B,MAAI,MAAK4B,0BACP;AAEF,aAAW,iBAAiB,WAAW,MAAKC,UAAW;AACvD,QAAKD,4BAA6B;;CAGpC,yBAA+B;AAC7B,MAAI,CAAC,MAAKA,0BACR;AAEF,aAAW,oBAAoB,WAAW,MAAKC,UAAW;AAC1D,QAAKD,4BAA6B;;CAGpC,qBAAqB,OAAqB;AAExC,aAAW,YAAY;GAAE,MAAM;GAAY;GAAO,CAAC;;CAGrD,4BACE,WACA,mBACA,oBAAoB,GACd;AACN,MAAI,sBAAsB,MAAKhC,aAAc,WAAW,CACtD;EAGF,MAAM,QAAQ,EAAE,MAAKI;AAErB,MAAI,MAAKU,MACP,SAAQ,IAAI,oDAAoD;GAC9D;GACA;GACA;GACA;GACD,CAAC;AAGJ,QAAKY,YAAa;AAClB,QAAKC,WAAY;AACjB,QAAKC,WAAY;AACjB,QAAKC,8BAA+B;AACpC,QAAKC,8BAA+B;AACpC,QAAKI,uBAAwB;AAC7B,QAAKH,oBAAqB,MAAM;;CAGlC,gBACE,MACA,OACgE;EAChE,MAAM,WAAW,MAAK/B,aAAc,YAAY,KAAK;AACrD,MAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,WAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAO;IAAE,gBAAgB,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC;IAAE;IAAO;;AAEvD,MACE,MAAKV,YAAa,UAClB,aAAa,MACb,SAAS,MAAM,KAAK,GAEpB,QAAO;GAAE,gBAAgB,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC;GAAE;GAAO;EAEvD,MAAM,SAAS,aACb,MAAKA,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,SAAO;GACL,gBAAgB,OAAO;GACvB,OAAO,OAAO;GACf;;CAGH,iBAAiB,OAAe;EAC9B,MAAM,eAAe,KAAK,IACxB,KAAK,IAAI,GAAG,MAAM,EAClB,MAAKf,aAAc,UACpB;AACD,MAAI,MAAKiB,WAAY,SAAS,gBAAgB,MAAKhB,YAAa,OAC9D;EAEF,IAAI,OAAO,MAAKgB,WAAY,SAAS;EACrC,IAAI,QAAQ,MAAKA,WAAY,SAAS;AACtC,SAAO,OAAO,cAAc,QAAQ;AAClC,SAAKA,WAAY,QAAQ;GACzB,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OACE,SAAS,UAAU,MAAKW,yBACxB,aAAa,MACb,SAAS,MAAM,KAAK,GAEpB,SAAQ,MAAKV,QAAS,cACpB,UACA,OACA,gBAAgB,oBACjB,CAAC;;AAGN,QAAKgB,WAAY,QAAQ;;CAG3B,oBAAoB,OAAe;AACjC,MACE,MAAKS,aACL,MAAKC,YACL,MAAK1B,YAAa,UAClB,UAAU,MAAKG,gBAEf;EAGF,MAAM,IAAI,YAAY,KAAK;EAC3B,MAAM,wBAAQ,IAAI,KAAsC;EACxD,MAAM,aAAa,MAAKJ,aAAc;EACtC,MAAM,oBAAoB,MAAK6B;EAE/B,IAAI,OAAO,MAAKD;EAChB,IAAI,QAAQ,MAAKX,WAAY,SAAS;EACtC,IAAI,UAAU;EACd,IAAI,oBAAoB,MAAKa;EAC7B,IAAI,yBAAyB,oBAAoB,qBAAqB;AACtE,SAAO,OAAO,aAAc;AAC1B,SAAKb,WAAY,QAAQ;GAEzB,MAAM,oBACJ,2BAA2B,SACvB,MAAKA,WAAY,OAAO,KACxB;GACN,MAAM,WAAW,MAAKjB,aAAc,YAAY,KAAK;AACrD,OAAI,SAAS,SAAS,MAAKW,uBAAwB;AACjD,YAAQ,KACN,gBAAgB,KAAK,0BAA0B,SAAS,SACzD;AACD,UAAM,IAAI,MAAM,CAAC;KAAC;KAAG;KAAI;KAAS,CAAC,CAAC;cAC3B,aAAa,MAAM,SAAS,MAAM,KAAK,GAChD,OAAM,IAAI,MAAM,CAAC;IAAC;IAAG;IAAI;IAAS,CAAC,CAAC;QAC/B;IACL,MAAM,MAAM,aACV,MAAKV,SACL,MAAKc,UACL,UACA,OACA,gBAAgB,oBACjB;AACD,UAAM,IAAI,MAAM,IAAI,eAAe;AACnC,YAAQ,IAAI;;AAGd,SAAKE,WAAY,OAAO,KAAK;AAC7B,aACE,2BAA2B,UAC3B,QAAQ,0BACR,sBAAsB,UACtB,MAAM,OAAO,kBAAkB;AACjC;AACA,OAAI,SAAS;AACX;IACA,MAAM,YAAY,oBAAoB;AACtC,QAAI,cAAc,OAChB;AAEF,6BAAyB,UAAU;AACnC,QAAI,MAAKA,WAAY,UAAU,QAAQ,OACrC,WAAU;SACL;AACL,YAAO,UAAU;AACjB,aAAQ,MAAKA,WAAY,SAAS;AAClC,eAAU;AACV;;;AAKJ,OAAI,YAAY,KAAK,GAAG,IAAI,EAC1B;;AAIJ,QAAKJ,gBAAiB,OAAO,MAAKP,UAAW;AAC7C,MAAI,MAAKoB,aAAc,MAAKC,YAAa,UAAU,MAAKvB,gBACtD;AAGF,MAAI,WAAW,QAAQ,YAAY;AACjC,QAAK,wBAAwB;AAC7B;;AAGF,QAAKwB,WAAY;AACjB,QAAKE,8BAA+B;AACpC,QAAKC,oBAAqB,MAAM;;;AAIpC,SAAgB,aACd,SACA,UACA,UACA,YACA,WAIA;CACA,MAAM,SAAS,QAAQ,cAAc,UAAU,YAAY,UAAU;AACrE,KAAI,OAAO,aACT,SAAQ,KACN,oDAAoD,SAAS,UAAU,GAAG,IAAI,GAC/E;CAEH,MAAM,YAAY,OAAO;CACzB,MAAM,eAAe,UAAU,SAAS;CACxC,MAAMI,iBAA0C,EAAE;AAClD,MAAK,IAAI,IAAI,GAAG,IAAI,cAAc,KAAK;EACrC,MAAM,SAAS,UAAU,IAAI;EAC7B,MAAM,aACJ,IAAI,IAAI,eAAe,UAAU,IAAI,IAAI,KAAK,SAAS;AACzD,MAAI,WAAW,WAEb;EAEF,MAAM,WAAW,UAAU,IAAI,IAAI;EAEnC,MAAM,KAAK,SADA,qBAAqB,cAAc,SAAS;EAEvD,MAAM,YAAY,SAAS,MAAM,QAAQ,WAAW;AACpD,iBAAe,KAAK;GAAC;GAAQ;GAAI;GAAU,CAAC;;AAE9C,QAAO;EACL,WAAW,OAAO;EAClB;EACD;;AAGH,SAAgB,iBACd,QACA,WAC0B;AAC1B,QAAO,OAAO,KAAK,CAAC,MAAM,IAAI,iBAAiB;AAC7C,MAAI,SAAS,KAAK,OAAO,IAAI;AAC3B,OAAI,gBAAgB,GAClB,QAAO,EAAE,KAAK;AAEhB,UAAO;;AAET,SAAO,EAAE,QAAQ;GACf,SAAS,EACP,MAAM,KAAK,UAAU,EACtB;GACD,OAAO,iBAAiB,UAAU,GAAG,GAAG;GAC3B;GACd,CAAC;GACF;;AAIJ,SAAS,sBAAsB,YAA6B;AAC1D,QAAO,eAAe,UAAU,eAAe"}

package/dist/managers/InteractionManager.js

@@ -1073,7 +1073,7 @@ function selectionPointFromAnnotationSlotName(slotName) {
 	const match = /^annotation-(?:(additions|deletions)-)?(\d+)$/.exec(slotName);
 	if (match == null) return;
 	const lineNumber = Number.parseInt(match[2], 10);
-	if (!Number.isFinite(lineNumber)) return;
+	if (!Number.isFinite(lineNumber) || lineNumber <= 0) return;
 	return {
 		lineNumber,
 		side: match[1]