@pierre/diffs 1.5.2 → 1.6.0-beta.1

package/dist/components/FileStream.d.ts

@@ -1,5 +1,6 @@
-import { BaseCodeOptions, SupportedLanguages, ThemeTypes, ThemedToken } from "../types.js";
-import { RecallToken } from "../shiki-stream/types.js";
+import { RecallToken } from "../highlighter/tokenizer-types.js";
+import { ThemedToken } from "../highlighter/types.js";
+import { BaseCodeOptions, SupportedLanguages, ThemeTypes } from "../types.js";
 import "../shiki-stream/index.js";
 //#region src/components/FileStream.d.ts
 interface FileStreamOptions extends BaseCodeOptions {
@@ -15,8 +16,8 @@ interface FileStreamOptions extends BaseCodeOptions {
 declare class FileStream {
   options: FileStreamOptions;
   readonly __id: string;
-  private highlighter;
   private stream;
+  private tokenStream;
   private abortController;
   private fileContainer;
   private pre;
@@ -29,7 +30,6 @@ declare class FileStream {
   constructor(options?: FileStreamOptions);
   cleanUp(): void;
   setThemeType(themeType: ThemeTypes): void;
-  private initializeHighlighter;
   private queuedSetupArgs;
   setup(_source: ReadableStream<string>, _wrapper: HTMLElement): Promise<void>;
   private setupStream;
@@ -37,6 +37,8 @@ declare class FileStream {
   private handleWrite;
   private currentLineIndex;
   private currentLineElement;
+  private pendingCarriageReturn;
+  private streamClosed;
   private render;
   private getOrCreateStreamColumns;
   private updateRowSpan;

package/dist/components/FileStream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"FileStream.d.ts","names":[],"sources":["../../src/components/FileStream.ts"],"mappings":";;;;UAyBiB,0BAA0B;EACzC,OAAO;EACP;EAEA,aAAa,UAAU;EACvB,cAAc,UAAU;EAExB,eAAe,YAAY;EAC3B,eAAe,OAAO,cAAc;EACpC;EACA,eAAe;;cAKJ;EAeQ,SAAS;WAdnB;UAED;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAmB,UAAS;EAI5B;EAMA,aAAa,WAAW;UAoBV;UAON;EACF,MACJ,SAAS,wBACT,UAAU,cACT;UAkBK;UA+FA;UACA;UAYA;UACA;UACA;UA8CA;UAqBA;UASA;UAwBA;UAoBA"}
+{"version":3,"file":"FileStream.d.ts","names":[],"sources":["../../src/components/FileStream.ts"],"mappings":";;;;;UAyBiB,0BAA0B;EACzC,OAAO;EACP;EAEA,aAAa,UAAU;EACvB,cAAc,UAAU;EAExB,eAAe,YAAY;EAC3B,eAAe,OAAO,cAAc;EACpC;EACA,eAAe;;cAKJ;EAeQ,SAAS;WAdnB;UAED;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAmB,UAAS;EAI5B;EAQA,aAAa,WAAW;UAoBhB;EACF,MACJ,SAAS,wBACT,UAAU,cACT;UAwBK;UA4FA;UACA;UAYA;UACA;UACA;UACA;UACA;UA4DA;UAqBA;UASA;UAwBA;UAoBA"}

package/dist/components/FileStream.js

@@ -2,8 +2,8 @@ import { DEFAULT_THEMES } from "../constants.js";
 import { getSharedHighlighter } from "../highlighter/shared_highlighter.js";
 import { dequeueRender, queueRender } from "../managers/UniversalRenderingManager.js";
 import { getHighlighterOptions } from "../utils/getHighlighterOptions.js";
-import { formatCSSVariablePrefix } from "../utils/formatCSSVariablePrefix.js";
 import { getHighlighterThemeStyles } from "../utils/getHighlighterThemeStyles.js";
+import { getTokenOptions } from "../utils/getTokenOptions.js";
 import { getMeasuredScrollbarGutter } from "../utils/scrollbarGutter.js";
 import { wrapThemeCSS } from "../utils/cssWrappers.js";
 import { getOrCreateCodeNode } from "../utils/getOrCreateCodeNode.js";
@@ -16,8 +16,8 @@ let instanceId = -1;
 var FileStream = class {
 	options;
 	__id = `file-stream:${++instanceId}`;
-	highlighter;
 	stream;
+	tokenStream;
 	abortController;
 	fileContainer;
 	pre;
@@ -35,6 +35,8 @@ var FileStream = class {
 		dequeueRender(this.render);
 		this.abortController?.abort();
 		this.abortController = void 0;
+		this.tokenStream?.tokenizer.dispose();
+		this.tokenStream = void 0;
 	}
 	setThemeType(themeType) {
 		if ((this.options.themeType ?? "system") === themeType) return;
@@ -45,20 +47,21 @@ var FileStream = class {
 		if (typeof this.options.theme === "string" || this.fileContainer == null || this.appliedThemeCSS == null) return;
 		this.applyThemeState(this.fileContainer, this.appliedThemeCSS.themeStyles, themeType, this.appliedThemeCSS.baseThemeType);
 	}
-	async initializeHighlighter() {
-		this.highlighter = await getSharedHighlighter(getHighlighterOptions(this.options.lang, this.options));
-		return this.highlighter;
-	}
 	queuedSetupArgs;
 	async setup(_source, _wrapper) {
 		const isSettingUp = this.queuedSetupArgs != null;
 		this.queuedSetupArgs = [_source, _wrapper];
 		if (isSettingUp) return;
-		this.highlighter ??= await this.initializeHighlighter();
+		let highlighter;
+		try {
+			highlighter = await getSharedHighlighter(getHighlighterOptions(this.options.lang, this.options));
+		} catch (error) {
+			this.queuedSetupArgs = void 0;
+			throw error;
+		}
 		const [source, wrapper] = this.queuedSetupArgs;
 		this.queuedSetupArgs = void 0;
-		const stream = source;
-		this.setupStream(stream, wrapper, this.highlighter);
+		this.setupStream(source, wrapper, highlighter);
 	}
 	setupStream(stream, wrapper, highlighter) {
 		const { disableLineNumbers = false, overflow = "scroll", theme = DEFAULT_THEMES, themeType = "system" } = this.options;
@@ -92,32 +95,30 @@ var FileStream = class {
 		this.currentRowCount = 0;
 		this.currentLineElement = void 0;
 		this.currentLineIndex = this.options.startingLineIndex ?? 1;
+		this.pendingCarriageReturn = false;
+		this.streamClosed = false;
 		this.abortController?.abort();
+		this.tokenStream?.tokenizer.dispose();
+		this.queuedTokens = [];
 		this.abortController = new AbortController();
 		const { onStreamStart, onStreamClose, onStreamAbort } = this.options;
 		this.stream?.cancel().catch(() => {});
 		this.stream = stream;
-		this.stream.pipeThrough(typeof theme === "string" ? new CodeToTokenTransformStream({
-			...this.options,
-			theme,
-			highlighter,
-			allowRecalls: true,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		}) : new CodeToTokenTransformStream({
-			...this.options,
-			themes: theme,
+		const tokenStream = new CodeToTokenTransformStream({
+			...getTokenOptions(this.options.lang ?? "text", theme, this.options.tokenizeMaxLineLength),
 			highlighter,
-			allowRecalls: true,
-			defaultColor: false,
-			cssVariablePrefix: formatCSSVariablePrefix("token"),
-			tokenizeTimeLimit: 0
-		})).pipeTo(new WritableStream({
+			allowRecalls: true
+		});
+		this.tokenStream = tokenStream;
+		this.stream.pipeThrough(tokenStream).pipeTo(new WritableStream({
 			start(controller) {
 				onStreamStart?.(controller);
 			},
-			close() {
+			close: () => {
+				if (this.tokenStream === tokenStream) {
+					this.streamClosed = true;
+					queueRender(this.render);
+				}
 				onStreamClose?.();
 			},
 			abort(reason) {
@@ -126,7 +127,7 @@ var FileStream = class {
 			write: this.handleWrite
 		}), { signal: this.abortController.signal }).catch((error) => {
 			if (error.name !== "AbortError") console.error("FileStream pipe error:", error);
-		});
+		}).finally(() => tokenStream.tokenizer.dispose());
 	}
 	queuedTokens = [];
 	handleWrite = (token) => {
@@ -137,30 +138,42 @@ var FileStream = class {
 	};
 	currentLineIndex;
 	currentLineElement;
+	pendingCarriageReturn = false;
+	streamClosed = false;
 	render = () => {
 		this.options.onPreRender?.(this);
 		const { gutter, content } = this.getOrCreateStreamColumns();
 		const gutterFragment = document.createDocumentFragment();
 		const contentFragment = document.createDocumentFragment();
+		const cacheHtmlStyle = this.tokenStream?.options.highlighter.name === "highlights";
+		const appendLine = () => {
+			const { gutterLine, contentLine } = this.createLine();
+			gutterFragment.appendChild(gutterLine);
+			contentFragment.appendChild(contentLine);
+		};
 		for (const token of this.queuedTokens) if ("recall" in token) {
 			if (this.currentLineElement == null) throw new Error("FileStream.render: no current line element, shouldnt be possible to get here");
 			if (token.recall > this.currentLineElement.childNodes.length) throw new Error(`FileStream.render: Token recall exceed the current line, there's probably a bug...`);
 			for (let i = 0; i < token.recall; i++) this.currentLineElement.lastChild?.remove();
+			if (token.recall > 0) this.pendingCarriageReturn = false;
 		} else {
-			const span = createSpanFromToken(token);
-			if (this.currentLineElement == null) {
-				const { gutterLine, contentLine } = this.createLine();
-				gutterFragment.appendChild(gutterLine);
-				contentFragment.appendChild(contentLine);
-			}
+			const span = createSpanFromToken(token, cacheHtmlStyle);
+			if (this.pendingCarriageReturn && token.content !== "\n") {
+				this.currentLineIndex++;
+				appendLine();
+			} else if (this.currentLineElement == null) appendLine();
 			this.currentLineElement?.appendChild(span);
+			this.pendingCarriageReturn = token.content === "\r";
 			if (token.content === "\n") {
 				this.currentLineIndex++;
-				const { gutterLine, contentLine } = this.createLine();
-				gutterFragment.appendChild(gutterLine);
-				contentFragment.appendChild(contentLine);
+				appendLine();
 			}
 		}
+		if (this.streamClosed && this.pendingCarriageReturn) {
+			this.pendingCarriageReturn = false;
+			this.currentLineIndex++;
+			appendLine();
+		}
 		if (gutterFragment.childNodes.length > 0) gutter.appendChild(gutterFragment);
 		if (contentFragment.childNodes.length > 0) content.appendChild(contentFragment);
 		this.queuedTokens.length = 0;

package/dist/components/FileStream.js.map

@@ -1 +1 @@
-{"version":3,"file":"FileStream.js","names":[],"sources":["../../src/components/FileStream.ts"],"sourcesContent":["import { DEFAULT_THEMES, DIFFS_TAG_NAME } from '../constants';\nimport { getSharedHighlighter } from '../highlighter/shared_highlighter';\nimport {\n  dequeueRender,\n  queueRender,\n} from '../managers/UniversalRenderingManager';\nimport { CodeToTokenTransformStream, type RecallToken } from '../shiki-stream';\nimport type {\n  AppliedThemeStyleCache,\n  BaseCodeOptions,\n  DiffsHighlighter,\n  SupportedLanguages,\n  ThemedToken,\n  ThemeTypes,\n} from '../types';\nimport { createSpanFromToken } from '../utils/createSpanNodeFromToken';\nimport { wrapThemeCSS } from '../utils/cssWrappers';\nimport { formatCSSVariablePrefix } from '../utils/formatCSSVariablePrefix';\nimport { getHighlighterOptions } from '../utils/getHighlighterOptions';\nimport { getHighlighterThemeStyles } from '../utils/getHighlighterThemeStyles';\nimport { getOrCreateCodeNode } from '../utils/getOrCreateCodeNode';\nimport { upsertHostThemeStyle } from '../utils/hostTheme';\nimport { getMeasuredScrollbarGutter } from '../utils/scrollbarGutter';\nimport { setPreNodeProperties } from '../utils/setWrapperNodeProps';\n\nexport interface FileStreamOptions extends BaseCodeOptions {\n  lang?: SupportedLanguages;\n  startingLineIndex?: number;\n\n  onPreRender?(instance: FileStream): unknown;\n  onPostRender?(instance: FileStream): unknown;\n\n  onStreamStart?(controller: WritableStreamDefaultController): unknown;\n  onStreamWrite?(token: ThemedToken | RecallToken): unknown;\n  onStreamClose?(): unknown;\n  onStreamAbort?(reason: unknown): unknown;\n}\n\nlet instanceId = -1;\n\nexport class FileStream {\n  readonly __id: string = `file-stream:${++instanceId}`;\n\n  private highlighter: DiffsHighlighter | undefined;\n  private stream: ReadableStream<string> | undefined;\n  private abortController: AbortController | undefined;\n  private fileContainer: HTMLElement | undefined;\n  private pre: HTMLPreElement | undefined;\n  private code: HTMLElement | undefined;\n  private gutterElement: HTMLElement | undefined;\n  private contentElement: HTMLElement | undefined;\n  private themeCSSStyle: HTMLStyleElement | undefined;\n  private appliedThemeCSS: AppliedThemeStyleCache | undefined;\n  private currentRowCount = 0;\n\n  constructor(public options: FileStreamOptions = { theme: DEFAULT_THEMES }) {\n    this.currentLineIndex = this.options.startingLineIndex ?? 1;\n  }\n\n  cleanUp(): void {\n    dequeueRender(this.render);\n    this.abortController?.abort();\n    this.abortController = undefined;\n  }\n\n  setThemeType(themeType: ThemeTypes): void {\n    if ((this.options.themeType ?? 'system') === themeType) {\n      return;\n    }\n    this.options = { ...this.options, themeType };\n    if (\n      typeof this.options.theme === 'string' ||\n      this.fileContainer == null ||\n      this.appliedThemeCSS == null\n    ) {\n      return;\n    }\n    this.applyThemeState(\n      this.fileContainer,\n      this.appliedThemeCSS.themeStyles,\n      themeType,\n      this.appliedThemeCSS.baseThemeType\n    );\n  }\n\n  private async initializeHighlighter(): Promise<DiffsHighlighter> {\n    this.highlighter = await getSharedHighlighter(\n      getHighlighterOptions(this.options.lang, this.options)\n    );\n    return this.highlighter;\n  }\n\n  private queuedSetupArgs: [ReadableStream<string>, HTMLElement] | undefined;\n  async setup(\n    _source: ReadableStream<string>,\n    _wrapper: HTMLElement\n  ): Promise<void> {\n    const isSettingUp = this.queuedSetupArgs != null;\n    this.queuedSetupArgs = [_source, _wrapper];\n    if (isSettingUp) {\n      // TODO(amadeus): Make it so that this function can be properly\n      // awaitable, maybe?\n      return;\n    }\n    this.highlighter ??= await this.initializeHighlighter();\n\n    const [source, wrapper] = this.queuedSetupArgs;\n    this.queuedSetupArgs = undefined;\n\n    const stream = source;\n\n    this.setupStream(stream, wrapper, this.highlighter);\n  }\n\n  private setupStream(\n    stream: ReadableStream<string>,\n    wrapper: HTMLElement,\n    highlighter: DiffsHighlighter\n  ): void {\n    const {\n      disableLineNumbers = false,\n      overflow = 'scroll',\n      theme = DEFAULT_THEMES,\n      themeType = 'system',\n    } = this.options;\n    const fileContainer = this.getOrCreateFileContainer();\n    if (fileContainer.parentElement == null) {\n      wrapper.appendChild(fileContainer);\n    }\n    this.pre ??= document.createElement('pre');\n    if (this.pre.parentElement == null) {\n      fileContainer.shadowRoot?.appendChild(this.pre);\n    }\n    const baseThemeType =\n      typeof theme === 'string' ? highlighter.getTheme(theme).type : undefined;\n    const themeStyles = getHighlighterThemeStyles({ theme, highlighter });\n    this.applyThemeState(fileContainer, themeStyles, themeType, baseThemeType);\n    const pre = setPreNodeProperties(this.pre, {\n      type: 'file',\n      diffIndicators: 'none',\n      disableBackground: true,\n      disableLineNumbers,\n      overflow,\n      split: false,\n      totalLines: 0,\n    });\n    pre.textContent = '';\n\n    this.pre = pre;\n    this.code = getOrCreateCodeNode({ code: this.code, pre });\n    this.gutterElement = undefined;\n    this.contentElement = undefined;\n    this.currentRowCount = 0;\n    this.currentLineElement = undefined;\n    this.currentLineIndex = this.options.startingLineIndex ?? 1;\n    this.abortController?.abort();\n    this.abortController = new AbortController();\n    const { onStreamStart, onStreamClose, onStreamAbort } = this.options;\n    // Cancel the prior source so upstream producers stop generating tokens.\n    // Swallow AbortError / locked-stream rejections since we're tearing down.\n    this.stream?.cancel().catch(() => {});\n    this.stream = stream;\n    this.stream\n      // tokenizeTimeLimit: 0 — never trade silently-wrong token colors for\n      // latency; see renderFileWithHighlighter for the full rationale.\n      .pipeThrough(\n        typeof theme === 'string'\n          ? new CodeToTokenTransformStream({\n              ...this.options,\n              theme,\n              highlighter,\n              allowRecalls: true,\n              defaultColor: false,\n              cssVariablePrefix: formatCSSVariablePrefix('token'),\n              tokenizeTimeLimit: 0,\n            })\n          : new CodeToTokenTransformStream({\n              ...this.options,\n              themes: theme,\n              highlighter,\n              allowRecalls: true,\n              defaultColor: false,\n              cssVariablePrefix: formatCSSVariablePrefix('token'),\n              tokenizeTimeLimit: 0,\n            })\n      )\n      .pipeTo(\n        new WritableStream({\n          start(controller) {\n            onStreamStart?.(controller);\n          },\n          close() {\n            onStreamClose?.();\n          },\n          abort(reason) {\n            onStreamAbort?.(reason);\n          },\n          write: this.handleWrite,\n        }),\n        { signal: this.abortController.signal }\n      )\n      .catch((error) => {\n        // Ignore AbortError - it's expected when cleaning up\n        if (error.name !== 'AbortError') {\n          console.error('FileStream pipe error:', error);\n        }\n      });\n  }\n\n  private queuedTokens: (ThemedToken | RecallToken)[] = [];\n  private handleWrite = (token: ThemedToken | RecallToken) => {\n    // If we've recalled tokens we haven't rendered yet, we can just yeet them\n    // and never apply them\n    if ('recall' in token && this.queuedTokens.length >= token.recall) {\n      this.queuedTokens.length = this.queuedTokens.length - token.recall;\n    } else {\n      this.queuedTokens.push(token);\n    }\n    queueRender(this.render);\n    this.options.onStreamWrite?.(token);\n  };\n\n  private currentLineIndex: number;\n  private currentLineElement: HTMLElement | undefined;\n  private render = () => {\n    this.options.onPreRender?.(this);\n    const { gutter, content } = this.getOrCreateStreamColumns();\n    const gutterFragment = document.createDocumentFragment();\n    const contentFragment = document.createDocumentFragment();\n    for (const token of this.queuedTokens) {\n      if ('recall' in token) {\n        if (this.currentLineElement == null) {\n          throw new Error(\n            'FileStream.render: no current line element, shouldnt be possible to get here'\n          );\n        }\n        if (token.recall > this.currentLineElement.childNodes.length) {\n          throw new Error(\n            `FileStream.render: Token recall exceed the current line, there's probably a bug...`\n          );\n        }\n        for (let i = 0; i < token.recall; i++) {\n          this.currentLineElement.lastChild?.remove();\n        }\n      } else {\n        const span = createSpanFromToken(token);\n        if (this.currentLineElement == null) {\n          const { gutterLine, contentLine } = this.createLine();\n          gutterFragment.appendChild(gutterLine);\n          contentFragment.appendChild(contentLine);\n        }\n        this.currentLineElement?.appendChild(span);\n        if (token.content === '\\n') {\n          this.currentLineIndex++;\n          const { gutterLine, contentLine } = this.createLine();\n          gutterFragment.appendChild(gutterLine);\n          contentFragment.appendChild(contentLine);\n        }\n      }\n    }\n    if (gutterFragment.childNodes.length > 0) {\n      gutter.appendChild(gutterFragment);\n    }\n    if (contentFragment.childNodes.length > 0) {\n      content.appendChild(contentFragment);\n    }\n    this.queuedTokens.length = 0;\n    this.options.onPostRender?.(this);\n  };\n\n  private getOrCreateStreamColumns(): {\n    gutter: HTMLElement;\n    content: HTMLElement;\n  } {\n    if (this.code == null) {\n      throw new Error('FileStream: expected code element to exist');\n    }\n    if (this.gutterElement != null && this.contentElement != null) {\n      return { gutter: this.gutterElement, content: this.contentElement };\n    }\n    const gutter = document.createElement('div');\n    gutter.dataset.gutter = '';\n    const content = document.createElement('div');\n    content.dataset.content = '';\n    this.code.appendChild(gutter);\n    this.code.appendChild(content);\n    this.gutterElement = gutter;\n    this.contentElement = content;\n    return { gutter, content };\n  }\n\n  private updateRowSpan(): void {\n    if (this.gutterElement != null) {\n      this.gutterElement.style.gridRow = `span ${this.currentRowCount}`;\n    }\n    if (this.contentElement != null) {\n      this.contentElement.style.gridRow = `span ${this.currentRowCount}`;\n    }\n  }\n\n  private createLine(): { gutterLine: HTMLElement; contentLine: HTMLElement } {\n    const lineNumber = this.currentLineIndex;\n    const lineIndex = `${lineNumber - 1}`;\n    const gutterLine = document.createElement('div');\n    gutterLine.dataset.columnNumber = `${lineNumber}`;\n    gutterLine.dataset.lineType = 'context';\n    gutterLine.dataset.lineIndex = lineIndex;\n\n    const numberContent = document.createElement('span');\n    numberContent.dataset.lineNumberContent = '';\n    numberContent.textContent = `${lineNumber}`;\n    gutterLine.appendChild(numberContent);\n\n    const contentLine = document.createElement('div');\n    contentLine.dataset.line = `${lineNumber}`;\n    contentLine.dataset.lineType = 'context';\n    contentLine.dataset.lineIndex = lineIndex;\n\n    this.currentRowCount += 1;\n    this.updateRowSpan();\n    this.currentLineElement = contentLine;\n    return { gutterLine, contentLine };\n  }\n\n  private getOrCreateFileContainer(fileContainer?: HTMLElement): HTMLElement {\n    if (\n      (fileContainer != null && fileContainer === this.fileContainer) ||\n      (fileContainer == null && this.fileContainer != null)\n    ) {\n      return this.fileContainer;\n    }\n    if (\n      this.fileContainer != null &&\n      fileContainer != null &&\n      fileContainer !== this.fileContainer\n    ) {\n      this.themeCSSStyle = undefined;\n      this.appliedThemeCSS = undefined;\n    }\n    this.fileContainer =\n      fileContainer ?? document.createElement(DIFFS_TAG_NAME);\n    return this.fileContainer;\n  }\n\n  private applyThemeState(\n    container: HTMLElement,\n    themeStyles: string,\n    themeType: ThemeTypes,\n    baseThemeType?: 'light' | 'dark'\n  ): void {\n    const shadowRoot =\n      container.shadowRoot ?? container.attachShadow({ mode: 'open' });\n    const effectiveThemeType = baseThemeType ?? themeType;\n    const currentTheme = this.options.theme ?? DEFAULT_THEMES;\n    const theme =\n      typeof currentTheme === 'string' ? currentTheme : { ...currentTheme };\n    const scrollbarGutter = getMeasuredScrollbarGutter(shadowRoot);\n    if (\n      this.themeCSSStyle?.parentNode === shadowRoot &&\n      this.appliedThemeCSS?.themeStyles === themeStyles &&\n      this.appliedThemeCSS.themeType === effectiveThemeType &&\n      this.appliedThemeCSS.scrollbarGutter === scrollbarGutter\n    ) {\n      this.appliedThemeCSS.theme = theme;\n      return;\n    }\n    this.themeCSSStyle = upsertHostThemeStyle({\n      shadowRoot,\n      currentNode: this.themeCSSStyle,\n      themeCSS: wrapThemeCSS(themeStyles, effectiveThemeType, scrollbarGutter),\n    });\n    this.appliedThemeCSS =\n      this.themeCSSStyle != null\n        ? {\n            theme,\n            themeStyles,\n            themeType: effectiveThemeType,\n            baseThemeType,\n            scrollbarGutter,\n          }\n        : undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAsCA,IAAI,aAAa;AAEjB,IAAa,aAAb,MAAwB;CAeH;CAdnB,OAAwB,eAAe,EAAE;CAEzC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,kBAA0B;CAE1B,YAAY,UAAoC,EAAE,OAAO,eAAe,GAAG;EAAxD,KAAA,UAAA;EACjB,KAAK,mBAAmB,KAAK,QAAQ,qBAAqB;CAC5D;CAEA,UAAgB;EACd,cAAc,KAAK,MAAM;EACzB,KAAK,iBAAiB,MAAM;EAC5B,KAAK,kBAAkB,KAAA;CACzB;CAEA,aAAa,WAA6B;EACxC,KAAK,KAAK,QAAQ,aAAa,cAAc,WAC3C;EAEF,KAAK,UAAU;GAAE,GAAG,KAAK;GAAS;EAAU;EAC5C,IACE,OAAO,KAAK,QAAQ,UAAU,YAC9B,KAAK,iBAAiB,QACtB,KAAK,mBAAmB,MAExB;EAEF,KAAK,gBACH,KAAK,eACL,KAAK,gBAAgB,aACrB,WACA,KAAK,gBAAgB,aACvB;CACF;CAEA,MAAc,wBAAmD;EAC/D,KAAK,cAAc,MAAM,qBACvB,sBAAsB,KAAK,QAAQ,MAAM,KAAK,OAAO,CACvD;EACA,OAAO,KAAK;CACd;CAEA;CACA,MAAM,MACJ,SACA,UACe;EACf,MAAM,cAAc,KAAK,mBAAmB;EAC5C,KAAK,kBAAkB,CAAC,SAAS,QAAQ;EACzC,IAAI,aAGF;EAEF,KAAK,gBAAgB,MAAM,KAAK,sBAAsB;EAEtD,MAAM,CAAC,QAAQ,WAAW,KAAK;EAC/B,KAAK,kBAAkB,KAAA;EAEvB,MAAM,SAAS;EAEf,KAAK,YAAY,QAAQ,SAAS,KAAK,WAAW;CACpD;CAEA,YACE,QACA,SACA,aACM;EACN,MAAM,EACJ,qBAAqB,OACrB,WAAW,UACX,QAAQ,gBACR,YAAY,aACV,KAAK;EACT,MAAM,gBAAgB,KAAK,yBAAyB;EACpD,IAAI,cAAc,iBAAiB,MACjC,QAAQ,YAAY,aAAa;EAEnC,KAAK,QAAQ,SAAS,cAAc,KAAK;EACzC,IAAI,KAAK,IAAI,iBAAiB,MAC5B,cAAc,YAAY,YAAY,KAAK,GAAG;EAEhD,MAAM,gBACJ,OAAO,UAAU,WAAW,YAAY,SAAS,KAAK,CAAC,CAAC,OAAO,KAAA;EACjE,MAAM,cAAc,0BAA0B;GAAE;GAAO;EAAY,CAAC;EACpE,KAAK,gBAAgB,eAAe,aAAa,WAAW,aAAa;EACzE,MAAM,MAAM,qBAAqB,KAAK,KAAK;GACzC,MAAM;GACN,gBAAgB;GAChB,mBAAmB;GACnB;GACA;GACA,OAAO;GACP,YAAY;EACd,CAAC;EACD,IAAI,cAAc;EAElB,KAAK,MAAM;EACX,KAAK,OAAO,oBAAoB;GAAE,MAAM,KAAK;GAAM;EAAI,CAAC;EACxD,KAAK,gBAAgB,KAAA;EACrB,KAAK,iBAAiB,KAAA;EACtB,KAAK,kBAAkB;EACvB,KAAK,qBAAqB,KAAA;EAC1B,KAAK,mBAAmB,KAAK,QAAQ,qBAAqB;EAC1D,KAAK,iBAAiB,MAAM;EAC5B,KAAK,kBAAkB,IAAI,gBAAgB;EAC3C,MAAM,EAAE,eAAe,eAAe,kBAAkB,KAAK;EAG7D,KAAK,QAAQ,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC;EACpC,KAAK,SAAS;EACd,KAAK,OAGF,YACC,OAAO,UAAU,WACb,IAAI,2BAA2B;GAC7B,GAAG,KAAK;GACR;GACA;GACA,cAAc;GACd,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD,mBAAmB;EACrB,CAAC,IACD,IAAI,2BAA2B;GAC7B,GAAG,KAAK;GACR,QAAQ;GACR;GACA,cAAc;GACd,cAAc;GACd,mBAAmB,wBAAwB,OAAO;GAClD,mBAAmB;EACrB,CAAC,CACP,CAAC,CACA,OACC,IAAI,eAAe;GACjB,MAAM,YAAY;IAChB,gBAAgB,UAAU;GAC5B;GACA,QAAQ;IACN,gBAAgB;GAClB;GACA,MAAM,QAAQ;IACZ,gBAAgB,MAAM;GACxB;GACA,OAAO,KAAK;EACd,CAAC,GACD,EAAE,QAAQ,KAAK,gBAAgB,OAAO,CACxC,CAAC,CACA,OAAO,UAAU;GAEhB,IAAI,MAAM,SAAS,cACjB,QAAQ,MAAM,0BAA0B,KAAK;EAEjD,CAAC;CACL;CAEA,eAAsD,CAAC;CACvD,eAAuB,UAAqC;EAG1D,IAAI,YAAY,SAAS,KAAK,aAAa,UAAU,MAAM,QACzD,KAAK,aAAa,SAAS,KAAK,aAAa,SAAS,MAAM;OAE5D,KAAK,aAAa,KAAK,KAAK;EAE9B,YAAY,KAAK,MAAM;EACvB,KAAK,QAAQ,gBAAgB,KAAK;CACpC;CAEA;CACA;CACA,eAAuB;EACrB,KAAK,QAAQ,cAAc,IAAI;EAC/B,MAAM,EAAE,QAAQ,YAAY,KAAK,yBAAyB;EAC1D,MAAM,iBAAiB,SAAS,uBAAuB;EACvD,MAAM,kBAAkB,SAAS,uBAAuB;EACxD,KAAK,MAAM,SAAS,KAAK,cACvB,IAAI,YAAY,OAAO;GACrB,IAAI,KAAK,sBAAsB,MAC7B,MAAM,IAAI,MACR,8EACF;GAEF,IAAI,MAAM,SAAS,KAAK,mBAAmB,WAAW,QACpD,MAAM,IAAI,MACR,oFACF;GAEF,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAChC,KAAK,mBAAmB,WAAW,OAAO;EAE9C,OAAO;GACL,MAAM,OAAO,oBAAoB,KAAK;GACtC,IAAI,KAAK,sBAAsB,MAAM;IACnC,MAAM,EAAE,YAAY,gBAAgB,KAAK,WAAW;IACpD,eAAe,YAAY,UAAU;IACrC,gBAAgB,YAAY,WAAW;GACzC;GACA,KAAK,oBAAoB,YAAY,IAAI;GACzC,IAAI,MAAM,YAAY,MAAM;IAC1B,KAAK;IACL,MAAM,EAAE,YAAY,gBAAgB,KAAK,WAAW;IACpD,eAAe,YAAY,UAAU;IACrC,gBAAgB,YAAY,WAAW;GACzC;EACF;EAEF,IAAI,eAAe,WAAW,SAAS,GACrC,OAAO,YAAY,cAAc;EAEnC,IAAI,gBAAgB,WAAW,SAAS,GACtC,QAAQ,YAAY,eAAe;EAErC,KAAK,aAAa,SAAS;EAC3B,KAAK,QAAQ,eAAe,IAAI;CAClC;CAEA,2BAGE;EACA,IAAI,KAAK,QAAQ,MACf,MAAM,IAAI,MAAM,4CAA4C;EAE9D,IAAI,KAAK,iBAAiB,QAAQ,KAAK,kBAAkB,MACvD,OAAO;GAAE,QAAQ,KAAK;GAAe,SAAS,KAAK;EAAe;EAEpE,MAAM,SAAS,SAAS,cAAc,KAAK;EAC3C,OAAO,QAAQ,SAAS;EACxB,MAAM,UAAU,SAAS,cAAc,KAAK;EAC5C,QAAQ,QAAQ,UAAU;EAC1B,KAAK,KAAK,YAAY,MAAM;EAC5B,KAAK,KAAK,YAAY,OAAO;EAC7B,KAAK,gBAAgB;EACrB,KAAK,iBAAiB;EACtB,OAAO;GAAE;GAAQ;EAAQ;CAC3B;CAEA,gBAA8B;EAC5B,IAAI,KAAK,iBAAiB,MACxB,KAAK,cAAc,MAAM,UAAU,QAAQ,KAAK;EAElD,IAAI,KAAK,kBAAkB,MACzB,KAAK,eAAe,MAAM,UAAU,QAAQ,KAAK;CAErD;CAEA,aAA4E;EAC1E,MAAM,aAAa,KAAK;EACxB,MAAM,YAAY,GAAG,aAAa;EAClC,MAAM,aAAa,SAAS,cAAc,KAAK;EAC/C,WAAW,QAAQ,eAAe,GAAG;EACrC,WAAW,QAAQ,WAAW;EAC9B,WAAW,QAAQ,YAAY;EAE/B,MAAM,gBAAgB,SAAS,cAAc,MAAM;EACnD,cAAc,QAAQ,oBAAoB;EAC1C,cAAc,cAAc,GAAG;EAC/B,WAAW,YAAY,aAAa;EAEpC,MAAM,cAAc,SAAS,cAAc,KAAK;EAChD,YAAY,QAAQ,OAAO,GAAG;EAC9B,YAAY,QAAQ,WAAW;EAC/B,YAAY,QAAQ,YAAY;EAEhC,KAAK,mBAAmB;EACxB,KAAK,cAAc;EACnB,KAAK,qBAAqB;EAC1B,OAAO;GAAE;GAAY;EAAY;CACnC;CAEA,yBAAiC,eAA0C;EACzE,IACG,iBAAiB,QAAQ,kBAAkB,KAAK,iBAChD,iBAAiB,QAAQ,KAAK,iBAAiB,MAEhD,OAAO,KAAK;EAEd,IACE,KAAK,iBAAiB,QACtB,iBAAiB,QACjB,kBAAkB,KAAK,eACvB;GACA,KAAK,gBAAgB,KAAA;GACrB,KAAK,kBAAkB,KAAA;EACzB;EACA,KAAK,gBACH,iBAAiB,SAAS,cAAA,iBAA4B;EACxD,OAAO,KAAK;CACd;CAEA,gBACE,WACA,aACA,WACA,eACM;EACN,MAAM,aACJ,UAAU,cAAc,UAAU,aAAa,EAAE,MAAM,OAAO,CAAC;EACjE,MAAM,qBAAqB,iBAAiB;EAC5C,MAAM,eAAe,KAAK,QAAQ,SAAS;EAC3C,MAAM,QACJ,OAAO,iBAAiB,WAAW,eAAe,EAAE,GAAG,aAAa;EACtE,MAAM,kBAAkB,2BAA2B,UAAU;EAC7D,IACE,KAAK,eAAe,eAAe,cACnC,KAAK,iBAAiB,gBAAgB,eACtC,KAAK,gBAAgB,cAAc,sBACnC,KAAK,gBAAgB,oBAAoB,iBACzC;GACA,KAAK,gBAAgB,QAAQ;GAC7B;EACF;EACA,KAAK,gBAAgB,qBAAqB;GACxC;GACA,aAAa,KAAK;GAClB,UAAU,aAAa,aAAa,oBAAoB,eAAe;EACzE,CAAC;EACD,KAAK,kBACH,KAAK,iBAAiB,OAClB;GACE;GACA;GACA,WAAW;GACX;GACA;EACF,IACA,KAAA;CACR;AACF"}
+{"version":3,"file":"FileStream.js","names":[],"sources":["../../src/components/FileStream.ts"],"sourcesContent":["import { DEFAULT_THEMES, DIFFS_TAG_NAME } from '../constants';\nimport { getSharedHighlighter } from '../highlighter/shared_highlighter';\nimport {\n  dequeueRender,\n  queueRender,\n} from '../managers/UniversalRenderingManager';\nimport { CodeToTokenTransformStream, type RecallToken } from '../shiki-stream';\nimport type {\n  AppliedThemeStyleCache,\n  BaseCodeOptions,\n  DiffsHighlighter,\n  SupportedLanguages,\n  ThemedToken,\n  ThemeTypes,\n} from '../types';\nimport { createSpanFromToken } from '../utils/createSpanNodeFromToken';\nimport { wrapThemeCSS } from '../utils/cssWrappers';\nimport { getHighlighterOptions } from '../utils/getHighlighterOptions';\nimport { getHighlighterThemeStyles } from '../utils/getHighlighterThemeStyles';\nimport { getOrCreateCodeNode } from '../utils/getOrCreateCodeNode';\nimport { getTokenOptions } from '../utils/getTokenOptions';\nimport { upsertHostThemeStyle } from '../utils/hostTheme';\nimport { getMeasuredScrollbarGutter } from '../utils/scrollbarGutter';\nimport { setPreNodeProperties } from '../utils/setWrapperNodeProps';\n\nexport interface FileStreamOptions extends BaseCodeOptions {\n  lang?: SupportedLanguages;\n  startingLineIndex?: number;\n\n  onPreRender?(instance: FileStream): unknown;\n  onPostRender?(instance: FileStream): unknown;\n\n  onStreamStart?(controller: WritableStreamDefaultController): unknown;\n  onStreamWrite?(token: ThemedToken | RecallToken): unknown;\n  onStreamClose?(): unknown;\n  onStreamAbort?(reason: unknown): unknown;\n}\n\nlet instanceId = -1;\n\nexport class FileStream {\n  readonly __id: string = `file-stream:${++instanceId}`;\n\n  private stream: ReadableStream<string> | undefined;\n  private tokenStream: CodeToTokenTransformStream | undefined;\n  private abortController: AbortController | undefined;\n  private fileContainer: HTMLElement | undefined;\n  private pre: HTMLPreElement | undefined;\n  private code: HTMLElement | undefined;\n  private gutterElement: HTMLElement | undefined;\n  private contentElement: HTMLElement | undefined;\n  private themeCSSStyle: HTMLStyleElement | undefined;\n  private appliedThemeCSS: AppliedThemeStyleCache | undefined;\n  private currentRowCount = 0;\n\n  constructor(public options: FileStreamOptions = { theme: DEFAULT_THEMES }) {\n    this.currentLineIndex = this.options.startingLineIndex ?? 1;\n  }\n\n  cleanUp(): void {\n    dequeueRender(this.render);\n    this.abortController?.abort();\n    this.abortController = undefined;\n    this.tokenStream?.tokenizer.dispose();\n    this.tokenStream = undefined;\n  }\n\n  setThemeType(themeType: ThemeTypes): void {\n    if ((this.options.themeType ?? 'system') === themeType) {\n      return;\n    }\n    this.options = { ...this.options, themeType };\n    if (\n      typeof this.options.theme === 'string' ||\n      this.fileContainer == null ||\n      this.appliedThemeCSS == null\n    ) {\n      return;\n    }\n    this.applyThemeState(\n      this.fileContainer,\n      this.appliedThemeCSS.themeStyles,\n      themeType,\n      this.appliedThemeCSS.baseThemeType\n    );\n  }\n\n  private queuedSetupArgs: [ReadableStream<string>, HTMLElement] | undefined;\n  async setup(\n    _source: ReadableStream<string>,\n    _wrapper: HTMLElement\n  ): Promise<void> {\n    const isSettingUp = this.queuedSetupArgs != null;\n    this.queuedSetupArgs = [_source, _wrapper];\n    if (isSettingUp) {\n      // TODO(amadeus): Make it so that this function can be properly\n      // awaitable, maybe?\n      return;\n    }\n    let highlighter: DiffsHighlighter;\n    try {\n      highlighter = await getSharedHighlighter(\n        getHighlighterOptions(this.options.lang, this.options)\n      );\n    } catch (error) {\n      this.queuedSetupArgs = undefined;\n      throw error;\n    }\n\n    const [source, wrapper] = this.queuedSetupArgs;\n    this.queuedSetupArgs = undefined;\n\n    this.setupStream(source, wrapper, highlighter);\n  }\n\n  private setupStream(\n    stream: ReadableStream<string>,\n    wrapper: HTMLElement,\n    highlighter: DiffsHighlighter\n  ): void {\n    const {\n      disableLineNumbers = false,\n      overflow = 'scroll',\n      theme = DEFAULT_THEMES,\n      themeType = 'system',\n    } = this.options;\n    const fileContainer = this.getOrCreateFileContainer();\n    if (fileContainer.parentElement == null) {\n      wrapper.appendChild(fileContainer);\n    }\n    this.pre ??= document.createElement('pre');\n    if (this.pre.parentElement == null) {\n      fileContainer.shadowRoot?.appendChild(this.pre);\n    }\n    const baseThemeType =\n      typeof theme === 'string' ? highlighter.getTheme(theme).type : undefined;\n    const themeStyles = getHighlighterThemeStyles({ theme, highlighter });\n    this.applyThemeState(fileContainer, themeStyles, themeType, baseThemeType);\n    const pre = setPreNodeProperties(this.pre, {\n      type: 'file',\n      diffIndicators: 'none',\n      disableBackground: true,\n      disableLineNumbers,\n      overflow,\n      split: false,\n      totalLines: 0,\n    });\n    pre.textContent = '';\n\n    this.pre = pre;\n    this.code = getOrCreateCodeNode({ code: this.code, pre });\n    this.gutterElement = undefined;\n    this.contentElement = undefined;\n    this.currentRowCount = 0;\n    this.currentLineElement = undefined;\n    this.currentLineIndex = this.options.startingLineIndex ?? 1;\n    this.pendingCarriageReturn = false;\n    this.streamClosed = false;\n    this.abortController?.abort();\n    this.tokenStream?.tokenizer.dispose();\n    this.queuedTokens = [];\n    this.abortController = new AbortController();\n    const { onStreamStart, onStreamClose, onStreamAbort } = this.options;\n    // Cancel the prior source so upstream producers stop generating tokens.\n    // Swallow AbortError / locked-stream rejections since we're tearing down.\n    this.stream?.cancel().catch(() => {});\n    this.stream = stream;\n    const tokenStream = new CodeToTokenTransformStream({\n      ...getTokenOptions(\n        this.options.lang ?? 'text',\n        theme,\n        this.options.tokenizeMaxLineLength\n      ),\n      highlighter,\n      allowRecalls: true,\n    });\n    this.tokenStream = tokenStream;\n    this.stream\n      .pipeThrough(tokenStream)\n      .pipeTo(\n        new WritableStream({\n          start(controller) {\n            onStreamStart?.(controller);\n          },\n          close: () => {\n            if (this.tokenStream === tokenStream) {\n              this.streamClosed = true;\n              queueRender(this.render);\n            }\n            onStreamClose?.();\n          },\n          abort(reason) {\n            onStreamAbort?.(reason);\n          },\n          write: this.handleWrite,\n        }),\n        { signal: this.abortController.signal }\n      )\n      .catch((error) => {\n        // Ignore AbortError - it's expected when cleaning up\n        if (error.name !== 'AbortError') {\n          console.error('FileStream pipe error:', error);\n        }\n      })\n      .finally(() => tokenStream.tokenizer.dispose());\n  }\n\n  private queuedTokens: (ThemedToken | RecallToken)[] = [];\n  private handleWrite = (token: ThemedToken | RecallToken) => {\n    // If we've recalled tokens we haven't rendered yet, we can just yeet them\n    // and never apply them\n    if ('recall' in token && this.queuedTokens.length >= token.recall) {\n      this.queuedTokens.length = this.queuedTokens.length - token.recall;\n    } else {\n      this.queuedTokens.push(token);\n    }\n    queueRender(this.render);\n    this.options.onStreamWrite?.(token);\n  };\n\n  private currentLineIndex: number;\n  private currentLineElement: HTMLElement | undefined;\n  private pendingCarriageReturn = false;\n  private streamClosed = false;\n  private render = () => {\n    this.options.onPreRender?.(this);\n    const { gutter, content } = this.getOrCreateStreamColumns();\n    const gutterFragment = document.createDocumentFragment();\n    const contentFragment = document.createDocumentFragment();\n    const cacheHtmlStyle =\n      this.tokenStream?.options.highlighter.name === 'highlights';\n    const appendLine = () => {\n      const { gutterLine, contentLine } = this.createLine();\n      gutterFragment.appendChild(gutterLine);\n      contentFragment.appendChild(contentLine);\n    };\n    for (const token of this.queuedTokens) {\n      if ('recall' in token) {\n        if (this.currentLineElement == null) {\n          throw new Error(\n            'FileStream.render: no current line element, shouldnt be possible to get here'\n          );\n        }\n        if (token.recall > this.currentLineElement.childNodes.length) {\n          throw new Error(\n            `FileStream.render: Token recall exceed the current line, there's probably a bug...`\n          );\n        }\n        for (let i = 0; i < token.recall; i++) {\n          this.currentLineElement.lastChild?.remove();\n        }\n        if (token.recall > 0) this.pendingCarriageReturn = false;\n      } else {\n        const span = createSpanFromToken(token, cacheHtmlStyle);\n        // Delay the CR's row break so recalls can remove it and CRLF counts once.\n        if (this.pendingCarriageReturn && token.content !== '\\n') {\n          this.currentLineIndex++;\n          appendLine();\n        } else if (this.currentLineElement == null) {\n          appendLine();\n        }\n        this.currentLineElement?.appendChild(span);\n        this.pendingCarriageReturn = token.content === '\\r';\n        if (token.content === '\\n') {\n          this.currentLineIndex++;\n          appendLine();\n        }\n      }\n    }\n    if (this.streamClosed && this.pendingCarriageReturn) {\n      this.pendingCarriageReturn = false;\n      this.currentLineIndex++;\n      appendLine();\n    }\n    if (gutterFragment.childNodes.length > 0) {\n      gutter.appendChild(gutterFragment);\n    }\n    if (contentFragment.childNodes.length > 0) {\n      content.appendChild(contentFragment);\n    }\n    this.queuedTokens.length = 0;\n    this.options.onPostRender?.(this);\n  };\n\n  private getOrCreateStreamColumns(): {\n    gutter: HTMLElement;\n    content: HTMLElement;\n  } {\n    if (this.code == null) {\n      throw new Error('FileStream: expected code element to exist');\n    }\n    if (this.gutterElement != null && this.contentElement != null) {\n      return { gutter: this.gutterElement, content: this.contentElement };\n    }\n    const gutter = document.createElement('div');\n    gutter.dataset.gutter = '';\n    const content = document.createElement('div');\n    content.dataset.content = '';\n    this.code.appendChild(gutter);\n    this.code.appendChild(content);\n    this.gutterElement = gutter;\n    this.contentElement = content;\n    return { gutter, content };\n  }\n\n  private updateRowSpan(): void {\n    if (this.gutterElement != null) {\n      this.gutterElement.style.gridRow = `span ${this.currentRowCount}`;\n    }\n    if (this.contentElement != null) {\n      this.contentElement.style.gridRow = `span ${this.currentRowCount}`;\n    }\n  }\n\n  private createLine(): { gutterLine: HTMLElement; contentLine: HTMLElement } {\n    const lineNumber = this.currentLineIndex;\n    const lineIndex = `${lineNumber - 1}`;\n    const gutterLine = document.createElement('div');\n    gutterLine.dataset.columnNumber = `${lineNumber}`;\n    gutterLine.dataset.lineType = 'context';\n    gutterLine.dataset.lineIndex = lineIndex;\n\n    const numberContent = document.createElement('span');\n    numberContent.dataset.lineNumberContent = '';\n    numberContent.textContent = `${lineNumber}`;\n    gutterLine.appendChild(numberContent);\n\n    const contentLine = document.createElement('div');\n    contentLine.dataset.line = `${lineNumber}`;\n    contentLine.dataset.lineType = 'context';\n    contentLine.dataset.lineIndex = lineIndex;\n\n    this.currentRowCount += 1;\n    this.updateRowSpan();\n    this.currentLineElement = contentLine;\n    return { gutterLine, contentLine };\n  }\n\n  private getOrCreateFileContainer(fileContainer?: HTMLElement): HTMLElement {\n    if (\n      (fileContainer != null && fileContainer === this.fileContainer) ||\n      (fileContainer == null && this.fileContainer != null)\n    ) {\n      return this.fileContainer;\n    }\n    if (\n      this.fileContainer != null &&\n      fileContainer != null &&\n      fileContainer !== this.fileContainer\n    ) {\n      this.themeCSSStyle = undefined;\n      this.appliedThemeCSS = undefined;\n    }\n    this.fileContainer =\n      fileContainer ?? document.createElement(DIFFS_TAG_NAME);\n    return this.fileContainer;\n  }\n\n  private applyThemeState(\n    container: HTMLElement,\n    themeStyles: string,\n    themeType: ThemeTypes,\n    baseThemeType?: 'light' | 'dark'\n  ): void {\n    const shadowRoot =\n      container.shadowRoot ?? container.attachShadow({ mode: 'open' });\n    const effectiveThemeType = baseThemeType ?? themeType;\n    const currentTheme = this.options.theme ?? DEFAULT_THEMES;\n    const theme =\n      typeof currentTheme === 'string' ? currentTheme : { ...currentTheme };\n    const scrollbarGutter = getMeasuredScrollbarGutter(shadowRoot);\n    if (\n      this.themeCSSStyle?.parentNode === shadowRoot &&\n      this.appliedThemeCSS?.themeStyles === themeStyles &&\n      this.appliedThemeCSS.themeType === effectiveThemeType &&\n      this.appliedThemeCSS.scrollbarGutter === scrollbarGutter\n    ) {\n      this.appliedThemeCSS.theme = theme;\n      return;\n    }\n    this.themeCSSStyle = upsertHostThemeStyle({\n      shadowRoot,\n      currentNode: this.themeCSSStyle,\n      themeCSS: wrapThemeCSS(themeStyles, effectiveThemeType, scrollbarGutter),\n    });\n    this.appliedThemeCSS =\n      this.themeCSSStyle != null\n        ? {\n            theme,\n            themeStyles,\n            themeType: effectiveThemeType,\n            baseThemeType,\n            scrollbarGutter,\n          }\n        : undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAsCA,IAAI,aAAa;AAEjB,IAAa,aAAb,MAAwB;CAeH;CAdnB,OAAwB,eAAe,EAAE;CAEzC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,kBAA0B;CAE1B,YAAY,UAAoC,EAAE,OAAO,eAAe,GAAG;EAAxD,KAAA,UAAA;EACjB,KAAK,mBAAmB,KAAK,QAAQ,qBAAqB;CAC5D;CAEA,UAAgB;EACd,cAAc,KAAK,MAAM;EACzB,KAAK,iBAAiB,MAAM;EAC5B,KAAK,kBAAkB,KAAA;EACvB,KAAK,aAAa,UAAU,QAAQ;EACpC,KAAK,cAAc,KAAA;CACrB;CAEA,aAAa,WAA6B;EACxC,KAAK,KAAK,QAAQ,aAAa,cAAc,WAC3C;EAEF,KAAK,UAAU;GAAE,GAAG,KAAK;GAAS;EAAU;EAC5C,IACE,OAAO,KAAK,QAAQ,UAAU,YAC9B,KAAK,iBAAiB,QACtB,KAAK,mBAAmB,MAExB;EAEF,KAAK,gBACH,KAAK,eACL,KAAK,gBAAgB,aACrB,WACA,KAAK,gBAAgB,aACvB;CACF;CAEA;CACA,MAAM,MACJ,SACA,UACe;EACf,MAAM,cAAc,KAAK,mBAAmB;EAC5C,KAAK,kBAAkB,CAAC,SAAS,QAAQ;EACzC,IAAI,aAGF;EAEF,IAAI;EACJ,IAAI;GACF,cAAc,MAAM,qBAClB,sBAAsB,KAAK,QAAQ,MAAM,KAAK,OAAO,CACvD;EACF,SAAS,OAAO;GACd,KAAK,kBAAkB,KAAA;GACvB,MAAM;EACR;EAEA,MAAM,CAAC,QAAQ,WAAW,KAAK;EAC/B,KAAK,kBAAkB,KAAA;EAEvB,KAAK,YAAY,QAAQ,SAAS,WAAW;CAC/C;CAEA,YACE,QACA,SACA,aACM;EACN,MAAM,EACJ,qBAAqB,OACrB,WAAW,UACX,QAAQ,gBACR,YAAY,aACV,KAAK;EACT,MAAM,gBAAgB,KAAK,yBAAyB;EACpD,IAAI,cAAc,iBAAiB,MACjC,QAAQ,YAAY,aAAa;EAEnC,KAAK,QAAQ,SAAS,cAAc,KAAK;EACzC,IAAI,KAAK,IAAI,iBAAiB,MAC5B,cAAc,YAAY,YAAY,KAAK,GAAG;EAEhD,MAAM,gBACJ,OAAO,UAAU,WAAW,YAAY,SAAS,KAAK,CAAC,CAAC,OAAO,KAAA;EACjE,MAAM,cAAc,0BAA0B;GAAE;GAAO;EAAY,CAAC;EACpE,KAAK,gBAAgB,eAAe,aAAa,WAAW,aAAa;EACzE,MAAM,MAAM,qBAAqB,KAAK,KAAK;GACzC,MAAM;GACN,gBAAgB;GAChB,mBAAmB;GACnB;GACA;GACA,OAAO;GACP,YAAY;EACd,CAAC;EACD,IAAI,cAAc;EAElB,KAAK,MAAM;EACX,KAAK,OAAO,oBAAoB;GAAE,MAAM,KAAK;GAAM;EAAI,CAAC;EACxD,KAAK,gBAAgB,KAAA;EACrB,KAAK,iBAAiB,KAAA;EACtB,KAAK,kBAAkB;EACvB,KAAK,qBAAqB,KAAA;EAC1B,KAAK,mBAAmB,KAAK,QAAQ,qBAAqB;EAC1D,KAAK,wBAAwB;EAC7B,KAAK,eAAe;EACpB,KAAK,iBAAiB,MAAM;EAC5B,KAAK,aAAa,UAAU,QAAQ;EACpC,KAAK,eAAe,CAAC;EACrB,KAAK,kBAAkB,IAAI,gBAAgB;EAC3C,MAAM,EAAE,eAAe,eAAe,kBAAkB,KAAK;EAG7D,KAAK,QAAQ,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC;EACpC,KAAK,SAAS;EACd,MAAM,cAAc,IAAI,2BAA2B;GACjD,GAAG,gBACD,KAAK,QAAQ,QAAQ,QACrB,OACA,KAAK,QAAQ,qBACf;GACA;GACA,cAAc;EAChB,CAAC;EACD,KAAK,cAAc;EACnB,KAAK,OACF,YAAY,WAAW,CAAC,CACxB,OACC,IAAI,eAAe;GACjB,MAAM,YAAY;IAChB,gBAAgB,UAAU;GAC5B;GACA,aAAa;IACX,IAAI,KAAK,gBAAgB,aAAa;KACpC,KAAK,eAAe;KACpB,YAAY,KAAK,MAAM;IACzB;IACA,gBAAgB;GAClB;GACA,MAAM,QAAQ;IACZ,gBAAgB,MAAM;GACxB;GACA,OAAO,KAAK;EACd,CAAC,GACD,EAAE,QAAQ,KAAK,gBAAgB,OAAO,CACxC,CAAC,CACA,OAAO,UAAU;GAEhB,IAAI,MAAM,SAAS,cACjB,QAAQ,MAAM,0BAA0B,KAAK;EAEjD,CAAC,CAAC,CACD,cAAc,YAAY,UAAU,QAAQ,CAAC;CAClD;CAEA,eAAsD,CAAC;CACvD,eAAuB,UAAqC;EAG1D,IAAI,YAAY,SAAS,KAAK,aAAa,UAAU,MAAM,QACzD,KAAK,aAAa,SAAS,KAAK,aAAa,SAAS,MAAM;OAE5D,KAAK,aAAa,KAAK,KAAK;EAE9B,YAAY,KAAK,MAAM;EACvB,KAAK,QAAQ,gBAAgB,KAAK;CACpC;CAEA;CACA;CACA,wBAAgC;CAChC,eAAuB;CACvB,eAAuB;EACrB,KAAK,QAAQ,cAAc,IAAI;EAC/B,MAAM,EAAE,QAAQ,YAAY,KAAK,yBAAyB;EAC1D,MAAM,iBAAiB,SAAS,uBAAuB;EACvD,MAAM,kBAAkB,SAAS,uBAAuB;EACxD,MAAM,iBACJ,KAAK,aAAa,QAAQ,YAAY,SAAS;EACjD,MAAM,mBAAmB;GACvB,MAAM,EAAE,YAAY,gBAAgB,KAAK,WAAW;GACpD,eAAe,YAAY,UAAU;GACrC,gBAAgB,YAAY,WAAW;EACzC;EACA,KAAK,MAAM,SAAS,KAAK,cACvB,IAAI,YAAY,OAAO;GACrB,IAAI,KAAK,sBAAsB,MAC7B,MAAM,IAAI,MACR,8EACF;GAEF,IAAI,MAAM,SAAS,KAAK,mBAAmB,WAAW,QACpD,MAAM,IAAI,MACR,oFACF;GAEF,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAChC,KAAK,mBAAmB,WAAW,OAAO;GAE5C,IAAI,MAAM,SAAS,GAAG,KAAK,wBAAwB;EACrD,OAAO;GACL,MAAM,OAAO,oBAAoB,OAAO,cAAc;GAEtD,IAAI,KAAK,yBAAyB,MAAM,YAAY,MAAM;IACxD,KAAK;IACL,WAAW;GACb,OAAO,IAAI,KAAK,sBAAsB,MACpC,WAAW;GAEb,KAAK,oBAAoB,YAAY,IAAI;GACzC,KAAK,wBAAwB,MAAM,YAAY;GAC/C,IAAI,MAAM,YAAY,MAAM;IAC1B,KAAK;IACL,WAAW;GACb;EACF;EAEF,IAAI,KAAK,gBAAgB,KAAK,uBAAuB;GACnD,KAAK,wBAAwB;GAC7B,KAAK;GACL,WAAW;EACb;EACA,IAAI,eAAe,WAAW,SAAS,GACrC,OAAO,YAAY,cAAc;EAEnC,IAAI,gBAAgB,WAAW,SAAS,GACtC,QAAQ,YAAY,eAAe;EAErC,KAAK,aAAa,SAAS;EAC3B,KAAK,QAAQ,eAAe,IAAI;CAClC;CAEA,2BAGE;EACA,IAAI,KAAK,QAAQ,MACf,MAAM,IAAI,MAAM,4CAA4C;EAE9D,IAAI,KAAK,iBAAiB,QAAQ,KAAK,kBAAkB,MACvD,OAAO;GAAE,QAAQ,KAAK;GAAe,SAAS,KAAK;EAAe;EAEpE,MAAM,SAAS,SAAS,cAAc,KAAK;EAC3C,OAAO,QAAQ,SAAS;EACxB,MAAM,UAAU,SAAS,cAAc,KAAK;EAC5C,QAAQ,QAAQ,UAAU;EAC1B,KAAK,KAAK,YAAY,MAAM;EAC5B,KAAK,KAAK,YAAY,OAAO;EAC7B,KAAK,gBAAgB;EACrB,KAAK,iBAAiB;EACtB,OAAO;GAAE;GAAQ;EAAQ;CAC3B;CAEA,gBAA8B;EAC5B,IAAI,KAAK,iBAAiB,MACxB,KAAK,cAAc,MAAM,UAAU,QAAQ,KAAK;EAElD,IAAI,KAAK,kBAAkB,MACzB,KAAK,eAAe,MAAM,UAAU,QAAQ,KAAK;CAErD;CAEA,aAA4E;EAC1E,MAAM,aAAa,KAAK;EACxB,MAAM,YAAY,GAAG,aAAa;EAClC,MAAM,aAAa,SAAS,cAAc,KAAK;EAC/C,WAAW,QAAQ,eAAe,GAAG;EACrC,WAAW,QAAQ,WAAW;EAC9B,WAAW,QAAQ,YAAY;EAE/B,MAAM,gBAAgB,SAAS,cAAc,MAAM;EACnD,cAAc,QAAQ,oBAAoB;EAC1C,cAAc,cAAc,GAAG;EAC/B,WAAW,YAAY,aAAa;EAEpC,MAAM,cAAc,SAAS,cAAc,KAAK;EAChD,YAAY,QAAQ,OAAO,GAAG;EAC9B,YAAY,QAAQ,WAAW;EAC/B,YAAY,QAAQ,YAAY;EAEhC,KAAK,mBAAmB;EACxB,KAAK,cAAc;EACnB,KAAK,qBAAqB;EAC1B,OAAO;GAAE;GAAY;EAAY;CACnC;CAEA,yBAAiC,eAA0C;EACzE,IACG,iBAAiB,QAAQ,kBAAkB,KAAK,iBAChD,iBAAiB,QAAQ,KAAK,iBAAiB,MAEhD,OAAO,KAAK;EAEd,IACE,KAAK,iBAAiB,QACtB,iBAAiB,QACjB,kBAAkB,KAAK,eACvB;GACA,KAAK,gBAAgB,KAAA;GACrB,KAAK,kBAAkB,KAAA;EACzB;EACA,KAAK,gBACH,iBAAiB,SAAS,cAAA,iBAA4B;EACxD,OAAO,KAAK;CACd;CAEA,gBACE,WACA,aACA,WACA,eACM;EACN,MAAM,aACJ,UAAU,cAAc,UAAU,aAAa,EAAE,MAAM,OAAO,CAAC;EACjE,MAAM,qBAAqB,iBAAiB;EAC5C,MAAM,eAAe,KAAK,QAAQ,SAAS;EAC3C,MAAM,QACJ,OAAO,iBAAiB,WAAW,eAAe,EAAE,GAAG,aAAa;EACtE,MAAM,kBAAkB,2BAA2B,UAAU;EAC7D,IACE,KAAK,eAAe,eAAe,cACnC,KAAK,iBAAiB,gBAAgB,eACtC,KAAK,gBAAgB,cAAc,sBACnC,KAAK,gBAAgB,oBAAoB,iBACzC;GACA,KAAK,gBAAgB,QAAQ;GAC7B;EACF;EACA,KAAK,gBAAgB,qBAAqB;GACxC;GACA,aAAa,KAAK;GAClB,UAAU,aAAa,aAAa,oBAAoB,eAAe;EACzE,CAAC;EACD,KAAK,kBACH,KAAK,iBAAiB,OAClB;GACE;GACA;GACA,WAAW;GACX;GACA;EACF,IACA,KAAA;CACR;AACF"}

package/dist/constants.d.ts

@@ -1,4 +1,4 @@
-import { CodeViewLayout, HunkExpansionRegion, RenderRange, SmoothScrollSettings, ThemesType, VirtualFileMetrics } from "./types.js";
+import { CodeViewLayout, HighlighterTypes, HunkExpansionRegion, RenderRange, SmoothScrollSettings, ThemesType, VirtualFileMetrics } from "./types.js";
 //#region src/constants.d.ts
 declare const DIFFS_TAG_NAME: 'diffs-container';
 declare const DIFFS_DEVELOPMENT_BUILD: boolean;
@@ -19,6 +19,7 @@ declare const HEADER_FILENAME_SUFFIX_SLOT_ID = "header-filename-suffix";
 declare const HEADER_METADATA_SLOT_ID = "header-metadata";
 declare const CUSTOM_HEADER_SLOT_ID = "header-custom";
 declare const DEFAULT_THEMES: ThemesType;
+declare const DEFAULT_HIGHLIGHTER: HighlighterTypes;
 declare const THEME_CSS_ATTRIBUTE = "data-theme-css";
 declare const UNSAFE_CSS_ATTRIBUTE = "data-unsafe-css";
 declare const CORE_CSS_ATTRIBUTE = "data-core-css";
@@ -36,5 +37,5 @@ declare const DEFAULT_EXPANDED_REGION: HunkExpansionRegion;
 declare const DEFAULT_RENDER_RANGE: RenderRange;
 declare const EMPTY_RENDER_RANGE: RenderRange;
 //#endregion
-export { CODE_VIEW_FOOTER_ATTRIBUTE, CODE_VIEW_HEADER_ATTRIBUTE, COMMIT_METADATA_SPLIT, CORE_CSS_ATTRIBUTE, CUSTOM_HEADER_SLOT_ID, DEFAULT_CODE_VIEW_FILE_METRICS, DEFAULT_CODE_VIEW_LAYOUT, DEFAULT_COLLAPSED_CONTEXT_THRESHOLD, DEFAULT_EXPANDED_REGION, DEFAULT_RENDER_RANGE, DEFAULT_SMOOTH_SCROLL_SETTINGS, DEFAULT_THEMES, DEFAULT_TOKENIZE_MAX_LENGTH, DEFAULT_VIRTUAL_FILE_METRICS, DIFFS_DEVELOPMENT_BUILD, DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY, DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE, DIFFS_TAG_NAME, EMPTY_RENDER_RANGE, FILENAME_HEADER_REGEX, FILE_CONTEXT_BLOB, GIT_DIFF_FILE_BREAK_REGEX, HEADER_FILENAME_SUFFIX_SLOT_ID, HEADER_METADATA_SLOT_ID, HEADER_PREFIX_SLOT_ID, HUNK_HEADER, INDEX_LINE_METADATA, MERGE_CONFLICT_BASE_MARKER_REGEX, MERGE_CONFLICT_END_MARKER_REGEX, MERGE_CONFLICT_SEPARATOR_MARKER_REGEX, MERGE_CONFLICT_START_MARKER_REGEX, SPLIT_WITH_NEWLINES, THEME_CSS_ATTRIBUTE, UNIFIED_DIFF_FILE_BREAK_REGEX, UNSAFE_CSS_ATTRIBUTE };
+export { CODE_VIEW_FOOTER_ATTRIBUTE, CODE_VIEW_HEADER_ATTRIBUTE, COMMIT_METADATA_SPLIT, CORE_CSS_ATTRIBUTE, CUSTOM_HEADER_SLOT_ID, DEFAULT_CODE_VIEW_FILE_METRICS, DEFAULT_CODE_VIEW_LAYOUT, DEFAULT_COLLAPSED_CONTEXT_THRESHOLD, DEFAULT_EXPANDED_REGION, DEFAULT_HIGHLIGHTER, DEFAULT_RENDER_RANGE, DEFAULT_SMOOTH_SCROLL_SETTINGS, DEFAULT_THEMES, DEFAULT_TOKENIZE_MAX_LENGTH, DEFAULT_VIRTUAL_FILE_METRICS, DIFFS_DEVELOPMENT_BUILD, DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY, DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE, DIFFS_TAG_NAME, EMPTY_RENDER_RANGE, FILENAME_HEADER_REGEX, FILE_CONTEXT_BLOB, GIT_DIFF_FILE_BREAK_REGEX, HEADER_FILENAME_SUFFIX_SLOT_ID, HEADER_METADATA_SLOT_ID, HEADER_PREFIX_SLOT_ID, HUNK_HEADER, INDEX_LINE_METADATA, MERGE_CONFLICT_BASE_MARKER_REGEX, MERGE_CONFLICT_END_MARKER_REGEX, MERGE_CONFLICT_SEPARATOR_MARKER_REGEX, MERGE_CONFLICT_START_MARKER_REGEX, SPLIT_WITH_NEWLINES, THEME_CSS_ATTRIBUTE, UNIFIED_DIFF_FILE_BREAK_REGEX, UNSAFE_CSS_ATTRIBUTE };
 //# sourceMappingURL=constants.d.ts.map

package/dist/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","names":[],"sources":["../src/constants.ts"],"mappings":";;cASa;cAIA;cASA,uBAAuB;cACvB,2BAA2B;cAC3B,+BAA+B;cAC/B,mBAAmB;cACnB,aAAa;cAEb,qBAAqB;cACrB,uBAAuB;cACvB,qBAAqB;cAGrB,mCAAmC;cACnC,kCAAkC;cAClC,uCAAuC;cACvC,iCAAiC;cAEjC;cACA;cACA;cACA;cAEA,gBAAgB;cAKhB;cACA;cACA;cACA;cACA;cACA;cACA;cAGA;cACA;cACA,8BAA8B;cAO9B,gCAAgC;cAKhC,0BAA0B;cAM1B,gCAAgC;cAMhC,yBAAyB;cAKzB,sBAAsB;cAOtB,oBAAoB"}
+{"version":3,"file":"constants.d.ts","names":[],"sources":["../src/constants.ts"],"mappings":";;cAUa;cAIA;cASA,uBAAuB;cACvB,2BAA2B;cAC3B,+BAA+B;cAC/B,mBAAmB;cACnB,aAAa;cAEb,qBAAqB;cACrB,uBAAuB;cACvB,qBAAqB;cAGrB,mCAAmC;cACnC,kCAAkC;cAClC,uCAAuC;cACvC,iCAAiC;cAEjC;cACA;cACA;cACA;cAEA,gBAAgB;cAMhB,qBAAqB;cAErB;cACA;cACA;cACA;cACA;cACA;cACA;cAGA;cACA;cACA,8BAA8B;cAO9B,gCAAgC;cAKhC,0BAA0B;cAM1B,gCAAgC;cAMhC,yBAAyB;cAKzB,sBAAsB;cAOtB,oBAAoB"}

package/dist/constants.js

@@ -27,6 +27,7 @@ const DEFAULT_THEMES = {
 	dark: "pierre-dark",
 	light: "pierre-light"
 };
+const DEFAULT_HIGHLIGHTER = "shiki-js";
 const THEME_CSS_ATTRIBUTE = "data-theme-css";
 const UNSAFE_CSS_ATTRIBUTE = "data-unsafe-css";
 const CORE_CSS_ATTRIBUTE = "data-core-css";
@@ -73,6 +74,6 @@ const EMPTY_RENDER_RANGE = {
 	bufferAfter: 0
 };
 //#endregion
-export { CODE_VIEW_FOOTER_ATTRIBUTE, CODE_VIEW_HEADER_ATTRIBUTE, COMMIT_METADATA_SPLIT, CORE_CSS_ATTRIBUTE, CUSTOM_HEADER_SLOT_ID, DEFAULT_CODE_VIEW_FILE_METRICS, DEFAULT_CODE_VIEW_LAYOUT, DEFAULT_COLLAPSED_CONTEXT_THRESHOLD, DEFAULT_EXPANDED_REGION, DEFAULT_RENDER_RANGE, DEFAULT_SMOOTH_SCROLL_SETTINGS, DEFAULT_THEMES, DEFAULT_TOKENIZE_MAX_LENGTH, DEFAULT_VIRTUAL_FILE_METRICS, DIFFS_DEVELOPMENT_BUILD, DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY, DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE, DIFFS_TAG_NAME, EMPTY_RENDER_RANGE, FILENAME_HEADER_REGEX, FILE_CONTEXT_BLOB, GIT_DIFF_FILE_BREAK_REGEX, HEADER_FILENAME_SUFFIX_SLOT_ID, HEADER_METADATA_SLOT_ID, HEADER_PREFIX_SLOT_ID, HUNK_HEADER, INDEX_LINE_METADATA, MERGE_CONFLICT_BASE_MARKER_REGEX, MERGE_CONFLICT_END_MARKER_REGEX, MERGE_CONFLICT_SEPARATOR_MARKER_REGEX, MERGE_CONFLICT_START_MARKER_REGEX, SPLIT_WITH_NEWLINES, THEME_CSS_ATTRIBUTE, UNIFIED_DIFF_FILE_BREAK_REGEX, UNSAFE_CSS_ATTRIBUTE };
+export { CODE_VIEW_FOOTER_ATTRIBUTE, CODE_VIEW_HEADER_ATTRIBUTE, COMMIT_METADATA_SPLIT, CORE_CSS_ATTRIBUTE, CUSTOM_HEADER_SLOT_ID, DEFAULT_CODE_VIEW_FILE_METRICS, DEFAULT_CODE_VIEW_LAYOUT, DEFAULT_COLLAPSED_CONTEXT_THRESHOLD, DEFAULT_EXPANDED_REGION, DEFAULT_HIGHLIGHTER, DEFAULT_RENDER_RANGE, DEFAULT_SMOOTH_SCROLL_SETTINGS, DEFAULT_THEMES, DEFAULT_TOKENIZE_MAX_LENGTH, DEFAULT_VIRTUAL_FILE_METRICS, DIFFS_DEVELOPMENT_BUILD, DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY, DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE, DIFFS_TAG_NAME, EMPTY_RENDER_RANGE, FILENAME_HEADER_REGEX, FILE_CONTEXT_BLOB, GIT_DIFF_FILE_BREAK_REGEX, HEADER_FILENAME_SUFFIX_SLOT_ID, HEADER_METADATA_SLOT_ID, HEADER_PREFIX_SLOT_ID, HUNK_HEADER, INDEX_LINE_METADATA, MERGE_CONFLICT_BASE_MARKER_REGEX, MERGE_CONFLICT_END_MARKER_REGEX, MERGE_CONFLICT_SEPARATOR_MARKER_REGEX, MERGE_CONFLICT_START_MARKER_REGEX, SPLIT_WITH_NEWLINES, THEME_CSS_ATTRIBUTE, UNIFIED_DIFF_FILE_BREAK_REGEX, UNSAFE_CSS_ATTRIBUTE };
 
 //# sourceMappingURL=constants.js.map

package/dist/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","names":[],"sources":["../src/constants.ts"],"sourcesContent":["import type {\n  CodeViewLayout,\n  HunkExpansionRegion,\n  RenderRange,\n  SmoothScrollSettings,\n  ThemesType,\n  VirtualFileMetrics,\n} from './types';\n\nexport const DIFFS_TAG_NAME = 'diffs-container' as const;\n\n// Keep this as a NODE_ENV read so app builds can hard-disable development-only\n// checks unless they are explicitly built for development.\nexport const DIFFS_DEVELOPMENT_BUILD: boolean = (() => {\n  try {\n    return process.env.NODE_ENV === 'development';\n  } catch {\n    return false;\n  }\n})();\n\n// Misc patch/content parsing regexes\nexport const COMMIT_METADATA_SPLIT: RegExp = /(?=^From [a-f0-9]+ .+$)/m;\nexport const GIT_DIFF_FILE_BREAK_REGEX: RegExp = /(?=^diff --git)/gm;\nexport const UNIFIED_DIFF_FILE_BREAK_REGEX: RegExp = /(?=^---\\s+\\S)/gm;\nexport const FILE_CONTEXT_BLOB: RegExp = /(?=^@@ )/gm;\nexport const HUNK_HEADER: RegExp =\n  /^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@(?: (.*))?/m;\nexport const SPLIT_WITH_NEWLINES: RegExp = /(?<=\\n)/;\nexport const FILENAME_HEADER_REGEX: RegExp = /^(---|\\+\\+\\+)\\s+([^\\t\\r\\n]+)/;\nexport const INDEX_LINE_METADATA: RegExp =\n  /^index ([0-9a-f]+)\\.\\.([0-9a-f]+)(?: (\\d+))?$/i;\n\nexport const MERGE_CONFLICT_START_MARKER_REGEX: RegExp = /^<{7,}(?:\\s.*)?$/;\nexport const MERGE_CONFLICT_BASE_MARKER_REGEX: RegExp = /^\\|{7,}(?:\\s.*)?$/;\nexport const MERGE_CONFLICT_SEPARATOR_MARKER_REGEX: RegExp = /^={7,}$/;\nexport const MERGE_CONFLICT_END_MARKER_REGEX: RegExp = /^>{7,}(?:\\s.*)?$/;\n\nexport const HEADER_PREFIX_SLOT_ID = 'header-prefix';\nexport const HEADER_FILENAME_SUFFIX_SLOT_ID = 'header-filename-suffix';\nexport const HEADER_METADATA_SLOT_ID = 'header-metadata';\nexport const CUSTOM_HEADER_SLOT_ID = 'header-custom';\n\nexport const DEFAULT_THEMES: ThemesType = {\n  dark: 'pierre-dark',\n  light: 'pierre-light',\n};\n\nexport const THEME_CSS_ATTRIBUTE = 'data-theme-css';\nexport const UNSAFE_CSS_ATTRIBUTE = 'data-unsafe-css';\nexport const CORE_CSS_ATTRIBUTE = 'data-core-css';\nexport const DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE = 'data-diffs-scrollbar-measure';\nexport const CODE_VIEW_HEADER_ATTRIBUTE = 'data-diffs-code-view-header';\nexport const CODE_VIEW_FOOTER_ATTRIBUTE = 'data-diffs-code-view-footer';\nexport const DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY =\n  '--diffs-scrollbar-gutter-measured';\n\nexport const DEFAULT_COLLAPSED_CONTEXT_THRESHOLD = 1;\nexport const DEFAULT_TOKENIZE_MAX_LENGTH = 100_000;\nexport const DEFAULT_VIRTUAL_FILE_METRICS: VirtualFileMetrics = {\n  hunkLineCount: 50,\n  lineHeight: 20,\n  diffHeaderHeight: 44,\n  spacing: 8,\n};\n\nexport const DEFAULT_CODE_VIEW_FILE_METRICS: VirtualFileMetrics = {\n  ...DEFAULT_VIRTUAL_FILE_METRICS,\n  hunkLineCount: 1,\n};\n\nexport const DEFAULT_CODE_VIEW_LAYOUT: CodeViewLayout = {\n  paddingTop: 8,\n  paddingBottom: 8,\n  gap: 8,\n};\n\nexport const DEFAULT_SMOOTH_SCROLL_SETTINGS: SmoothScrollSettings = {\n  omega: 0.015,\n  positionEpsilon: 0.5,\n  velocityEpsilon: 0.05,\n};\n\nexport const DEFAULT_EXPANDED_REGION: HunkExpansionRegion = Object.freeze({\n  fromStart: 0,\n  fromEnd: 0,\n});\n\nexport const DEFAULT_RENDER_RANGE: RenderRange = {\n  startingLine: 0,\n  totalLines: Infinity,\n  bufferBefore: 0,\n  bufferAfter: 0,\n};\n\nexport const EMPTY_RENDER_RANGE: RenderRange = {\n  startingLine: 0,\n  totalLines: 0,\n  bufferBefore: 0,\n  bufferAfter: 0,\n};\n"],"mappings":";AASA,MAAa,iBAAiB;AAI9B,MAAa,iCAA0C;CACrD,IAAI;EACF,OAAO,QAAQ,IAAI,aAAa;CAClC,QAAQ;EACN,OAAO;CACT;AACF,EAAA,CAAG;AAGH,MAAa,wBAAgC;AAC7C,MAAa,4BAAoC;AACjD,MAAa,gCAAwC;AACrD,MAAa,oBAA4B;AACzC,MAAa,cACX;AACF,MAAa,sBAA8B;AAC3C,MAAa,wBAAgC;AAC7C,MAAa,sBACX;AAEF,MAAa,oCAA4C;AACzD,MAAa,mCAA2C;AACxD,MAAa,wCAAgD;AAC7D,MAAa,kCAA0C;AAEvD,MAAa,wBAAwB;AACrC,MAAa,iCAAiC;AAC9C,MAAa,0BAA0B;AACvC,MAAa,wBAAwB;AAErC,MAAa,iBAA6B;CACxC,MAAM;CACN,OAAO;AACT;AAEA,MAAa,sBAAsB;AACnC,MAAa,uBAAuB;AACpC,MAAa,qBAAqB;AAClC,MAAa,oCAAoC;AACjD,MAAa,6BAA6B;AAC1C,MAAa,6BAA6B;AAC1C,MAAa,2CACX;AAEF,MAAa,sCAAsC;AACnD,MAAa,8BAA8B;AAC3C,MAAa,+BAAmD;CAC9D,eAAe;CACf,YAAY;CACZ,kBAAkB;CAClB,SAAS;AACX;AAEA,MAAa,iCAAqD;CAChE,GAAG;CACH,eAAe;AACjB;AAEA,MAAa,2BAA2C;CACtD,YAAY;CACZ,eAAe;CACf,KAAK;AACP;AAEA,MAAa,iCAAuD;CAClE,OAAO;CACP,iBAAiB;CACjB,iBAAiB;AACnB;AAEA,MAAa,0BAA+C,OAAO,OAAO;CACxE,WAAW;CACX,SAAS;AACX,CAAC;AAED,MAAa,uBAAoC;CAC/C,cAAc;CACd,YAAY;CACZ,cAAc;CACd,aAAa;AACf;AAEA,MAAa,qBAAkC;CAC7C,cAAc;CACd,YAAY;CACZ,cAAc;CACd,aAAa;AACf"}
+{"version":3,"file":"constants.js","names":[],"sources":["../src/constants.ts"],"sourcesContent":["import type {\n  CodeViewLayout,\n  HighlighterTypes,\n  HunkExpansionRegion,\n  RenderRange,\n  SmoothScrollSettings,\n  ThemesType,\n  VirtualFileMetrics,\n} from './types';\n\nexport const DIFFS_TAG_NAME = 'diffs-container' as const;\n\n// Keep this as a NODE_ENV read so app builds can hard-disable development-only\n// checks unless they are explicitly built for development.\nexport const DIFFS_DEVELOPMENT_BUILD: boolean = (() => {\n  try {\n    return process.env.NODE_ENV === 'development';\n  } catch {\n    return false;\n  }\n})();\n\n// Misc patch/content parsing regexes\nexport const COMMIT_METADATA_SPLIT: RegExp = /(?=^From [a-f0-9]+ .+$)/m;\nexport const GIT_DIFF_FILE_BREAK_REGEX: RegExp = /(?=^diff --git)/gm;\nexport const UNIFIED_DIFF_FILE_BREAK_REGEX: RegExp = /(?=^---\\s+\\S)/gm;\nexport const FILE_CONTEXT_BLOB: RegExp = /(?=^@@ )/gm;\nexport const HUNK_HEADER: RegExp =\n  /^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@(?: (.*))?/m;\nexport const SPLIT_WITH_NEWLINES: RegExp = /(?<=\\n)/;\nexport const FILENAME_HEADER_REGEX: RegExp = /^(---|\\+\\+\\+)\\s+([^\\t\\r\\n]+)/;\nexport const INDEX_LINE_METADATA: RegExp =\n  /^index ([0-9a-f]+)\\.\\.([0-9a-f]+)(?: (\\d+))?$/i;\n\nexport const MERGE_CONFLICT_START_MARKER_REGEX: RegExp = /^<{7,}(?:\\s.*)?$/;\nexport const MERGE_CONFLICT_BASE_MARKER_REGEX: RegExp = /^\\|{7,}(?:\\s.*)?$/;\nexport const MERGE_CONFLICT_SEPARATOR_MARKER_REGEX: RegExp = /^={7,}$/;\nexport const MERGE_CONFLICT_END_MARKER_REGEX: RegExp = /^>{7,}(?:\\s.*)?$/;\n\nexport const HEADER_PREFIX_SLOT_ID = 'header-prefix';\nexport const HEADER_FILENAME_SUFFIX_SLOT_ID = 'header-filename-suffix';\nexport const HEADER_METADATA_SLOT_ID = 'header-metadata';\nexport const CUSTOM_HEADER_SLOT_ID = 'header-custom';\n\nexport const DEFAULT_THEMES: ThemesType = {\n  dark: 'pierre-dark',\n  light: 'pierre-light',\n};\n\n// The highlighter type used when no request names one and none is active yet.\nexport const DEFAULT_HIGHLIGHTER: HighlighterTypes = 'shiki-js';\n\nexport const THEME_CSS_ATTRIBUTE = 'data-theme-css';\nexport const UNSAFE_CSS_ATTRIBUTE = 'data-unsafe-css';\nexport const CORE_CSS_ATTRIBUTE = 'data-core-css';\nexport const DIFFS_SCROLLBAR_MEASURE_ATTRIBUTE = 'data-diffs-scrollbar-measure';\nexport const CODE_VIEW_HEADER_ATTRIBUTE = 'data-diffs-code-view-header';\nexport const CODE_VIEW_FOOTER_ATTRIBUTE = 'data-diffs-code-view-footer';\nexport const DIFFS_SCROLLBAR_GUTTER_MEASURED_PROPERTY =\n  '--diffs-scrollbar-gutter-measured';\n\nexport const DEFAULT_COLLAPSED_CONTEXT_THRESHOLD = 1;\nexport const DEFAULT_TOKENIZE_MAX_LENGTH = 100_000;\nexport const DEFAULT_VIRTUAL_FILE_METRICS: VirtualFileMetrics = {\n  hunkLineCount: 50,\n  lineHeight: 20,\n  diffHeaderHeight: 44,\n  spacing: 8,\n};\n\nexport const DEFAULT_CODE_VIEW_FILE_METRICS: VirtualFileMetrics = {\n  ...DEFAULT_VIRTUAL_FILE_METRICS,\n  hunkLineCount: 1,\n};\n\nexport const DEFAULT_CODE_VIEW_LAYOUT: CodeViewLayout = {\n  paddingTop: 8,\n  paddingBottom: 8,\n  gap: 8,\n};\n\nexport const DEFAULT_SMOOTH_SCROLL_SETTINGS: SmoothScrollSettings = {\n  omega: 0.015,\n  positionEpsilon: 0.5,\n  velocityEpsilon: 0.05,\n};\n\nexport const DEFAULT_EXPANDED_REGION: HunkExpansionRegion = Object.freeze({\n  fromStart: 0,\n  fromEnd: 0,\n});\n\nexport const DEFAULT_RENDER_RANGE: RenderRange = {\n  startingLine: 0,\n  totalLines: Infinity,\n  bufferBefore: 0,\n  bufferAfter: 0,\n};\n\nexport const EMPTY_RENDER_RANGE: RenderRange = {\n  startingLine: 0,\n  totalLines: 0,\n  bufferBefore: 0,\n  bufferAfter: 0,\n};\n"],"mappings":";AAUA,MAAa,iBAAiB;AAI9B,MAAa,iCAA0C;CACrD,IAAI;EACF,OAAO,QAAQ,IAAI,aAAa;CAClC,QAAQ;EACN,OAAO;CACT;AACF,EAAA,CAAG;AAGH,MAAa,wBAAgC;AAC7C,MAAa,4BAAoC;AACjD,MAAa,gCAAwC;AACrD,MAAa,oBAA4B;AACzC,MAAa,cACX;AACF,MAAa,sBAA8B;AAC3C,MAAa,wBAAgC;AAC7C,MAAa,sBACX;AAEF,MAAa,oCAA4C;AACzD,MAAa,mCAA2C;AACxD,MAAa,wCAAgD;AAC7D,MAAa,kCAA0C;AAEvD,MAAa,wBAAwB;AACrC,MAAa,iCAAiC;AAC9C,MAAa,0BAA0B;AACvC,MAAa,wBAAwB;AAErC,MAAa,iBAA6B;CACxC,MAAM;CACN,OAAO;AACT;AAGA,MAAa,sBAAwC;AAErD,MAAa,sBAAsB;AACnC,MAAa,uBAAuB;AACpC,MAAa,qBAAqB;AAClC,MAAa,oCAAoC;AACjD,MAAa,6BAA6B;AAC1C,MAAa,6BAA6B;AAC1C,MAAa,2CACX;AAEF,MAAa,sCAAsC;AACnD,MAAa,8BAA8B;AAC3C,MAAa,+BAAmD;CAC9D,eAAe;CACf,YAAY;CACZ,kBAAkB;CAClB,SAAS;AACX;AAEA,MAAa,iCAAqD;CAChE,GAAG;CACH,eAAe;AACjB;AAEA,MAAa,2BAA2C;CACtD,YAAY;CACZ,eAAe;CACf,KAAK;AACP;AAEA,MAAa,iCAAuD;CAClE,OAAO;CACP,iBAAiB;CACjB,iBAAiB;AACnB;AAEA,MAAa,0BAA+C,OAAO,OAAO;CACxE,WAAW;CACX,SAAS;AACX,CAAC;AAED,MAAa,uBAAoC;CAC/C,cAAc;CACd,YAAY;CACZ,cAAc;CACd,aAAa;AACf;AAEA,MAAa,qBAAkC;CAC7C,cAAc;CACd,YAAY;CACZ,cAAc;CACd,aAAa;AACf"}

package/dist/editor/editor.d.ts

@@ -1,4 +1,5 @@
-import { DiffLineAnnotation, DiffsHighlighter, FileContents, FileDiffMetadata, LineAnnotation, RenderRange } from "../types.js";
+import { DiffsHighlighter } from "../highlighter/DiffsHighlighter.js";
+import { DiffLineAnnotation, FileContents, FileDiffMetadata, LineAnnotation, RenderRange } from "../types.js";
 import { EditState, EditorCaret, EditorChangeEvent, EditorEditCompleteEvent, EditorInitialState, EditorType, EditorViewState, Range, RetainedDiffSessionSnapshot, TextEdit } from "./types.js";
 import { File } from "../components/File.js";
 import { FileDiff } from "../components/FileDiff.js";

package/dist/editor/editor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"editor.d.ts","names":[],"sources":["../../src/editor/editor.ts"],"mappings":";;;;;;;;;;;UAyOU;EACR,aAAa;EACb,eAAe;EACf,aAAa;;EAEb;;UAGQ,wBAAwB,qBAAqB;EACrD,MAAM;EACN,iBAAiB,eAAe;;EAEhC;;UAGQ,wBAAwB,qBAAqB;EACrD,UAAU;EACV,iBAAiB,mBAAmB;;EAEpC;;KAGG,oBACH,cAAc,YACd,eACE,uBACA,wBAAwB,eACxB,wBAAwB;KAEvB,gBACH,cAAc,YACd,aACA,SACE,uBACA,KAAK,aAAa,SAClB,SAAS,aAAa;UAkBT,cAAc,cAAc,YAAY,aAAa;;EAEpE;;;;;;EAMA;;;;;;EAMA,eAAe,mBAAmB,OAAO;;EAEzC,SAAS;;EAET;;EAEA;;;;;EAKA,eAAe;;EAEf,wBAAwB;;;;;EAKxB;;;;EAIA;;;;;;;IAOE;;;;IAIA,UAAU;;;;;;IAMV,6BAA6B;;;;;;IAM7B,6BAA6B;;;;;;;EAO/B;IACE,WAAW,kBAAkB;;;EAG/B,yBACE,SAAS,uBAAuB,OAAO,iBACpC;;;;EAIL,eAAe,OAAO,YAAY,WAAW;;EAE7C,YACE,QAAQ,OAAO,OAAO,aAAa,QACnC,cAAc,gBAAgB,OAAO,aAAa;;;;;;EAOpD,YAAY,OAAO,kBAAkB,OAAO,aAAa;;;;;;EAMzD,cACE,OAAO,wBAAwB,OAAO,aAAa;;EAGrD;;EAEA;EAEA;;UAGe,2BAA2B;;EAE1C;;EAEA;;EAEA;;cAiDW,OACX,cAAc,aAAa,YAC3B,yBACA;;WAyMkB,MAAM;;;;;;;EADxB,YACkB,MAAM,OACtB,UAAS,cAAc,OAAO,aAAa,QAC3C;EAkBF,WAAW,SAAS,cAAc,OAAO,aAAa;EAmBtD,mBACE,OAAO,wBAAwB,OAAO,aAAa;;EAMrD,sBAAsB;EAItB,UAAU,QAAQ,YAAY;EA6B9B,KAAK,UAAU,gBAAgB,OAAO,aAAa,QACjD,cAAc;aAAqB;cAAoC;;;;;;;;;;;EA4IzE,WAAW,OAAO,YAAY;;MAmE1B;;MAOA;;EAOJ;;EAKA;EAIA,WAAW;EAeX;;EAKA,gBAAgB;;;;;EA6BhB,gBAAgB,UAAU,OAAO;EAcjC,eAAe,YAAY,QAAQ;EA2BnC,cACE,aAAa;IAAU;;EAoCzB,WAAW,SAAS;EA8BpB,MAAM,UAAU;EAuChB;EAiBA,QAAQ;;EA2JR;;EAWA;;EAeA,sBAAsB,eAAe,eAAe;;EAoBpD,6BAA6B;;EAQ7B,iBAAiB,OAAO,oBAAoB,OAAO"}
+{"version":3,"file":"editor.d.ts","names":[],"sources":["../../src/editor/editor.ts"],"mappings":";;;;;;;;;;;;UAyOU;EACR,aAAa;EACb,eAAe;EACf,aAAa;;EAEb;;UAGQ,wBAAwB,qBAAqB;EACrD,MAAM;EACN,iBAAiB,eAAe;;EAEhC;;UAGQ,wBAAwB,qBAAqB;EACrD,UAAU;EACV,iBAAiB,mBAAmB;;EAEpC;;KAGG,oBACH,cAAc,YACd,eACE,uBACA,wBAAwB,eACxB,wBAAwB;KAEvB,gBACH,cAAc,YACd,aACA,SACE,uBACA,KAAK,aAAa,SAClB,SAAS,aAAa;UAkBT,cAAc,cAAc,YAAY,aAAa;;EAEpE;;;;;;EAMA;;;;;;EAMA,eAAe,mBAAmB,OAAO;;EAEzC,SAAS;;EAET;;EAEA;;;;;EAKA,eAAe;;EAEf,wBAAwB;;;;;EAKxB;;;;EAIA;;;;;;;IAOE;;;;IAIA,UAAU;;;;;;IAMV,6BAA6B;;;;;;IAM7B,6BAA6B;;;;;;;EAO/B;IACE,WAAW,kBAAkB;;;EAG/B,yBACE,SAAS,uBAAuB,OAAO,iBACpC;;;;EAIL,eAAe,OAAO,YAAY,WAAW;;EAE7C,YACE,QAAQ,OAAO,OAAO,aAAa,QACnC,cAAc,gBAAgB,OAAO,aAAa;;;;;;EAOpD,YAAY,OAAO,kBAAkB,OAAO,aAAa;;;;;;EAMzD,cACE,OAAO,wBAAwB,OAAO,aAAa;;EAGrD;;EAEA;EAEA;;UAGe,2BAA2B;;EAE1C;;EAEA;;EAEA;;cAiDW,OACX,cAAc,aAAa,YAC3B,yBACA;;WAyMkB,MAAM;;;;;;;EADxB,YACkB,MAAM,OACtB,UAAS,cAAc,OAAO,aAAa,QAC3C;EAkBF,WAAW,SAAS,cAAc,OAAO,aAAa;EAmBtD,mBACE,OAAO,wBAAwB,OAAO,aAAa;;EAMrD,sBAAsB;EAItB,UAAU,QAAQ,YAAY;EA6B9B,KAAK,UAAU,gBAAgB,OAAO,aAAa,QACjD,cAAc;aAAqB;cAAoC;;;;;;;;;;;EA4IzE,WAAW,OAAO,YAAY;;MAmE1B;;MAOA;;EAOJ;;EAKA;EAIA,WAAW;EAeX;;EAKA,gBAAgB;;;;;EA6BhB,gBAAgB,UAAU,OAAO;EAcjC,eAAe,YAAY,QAAQ;EA2BnC,cACE,aAAa;IAAU;;EAoCzB,WAAW,SAAS;EA8BpB,MAAM,UAAU;EAuChB;EAiBA,QAAQ;;EA2JR;;EAaA;;EAeA,sBAAsB,eAAe,eAAe;;EAoBpD,6BAA6B;;EAQ7B,iBAAiB,OAAO,oBAAoB,OAAO"}

package/dist/editor/editor.js

@@ -594,7 +594,7 @@ var Editor = class {
 		if (tokenizer !== void 0) {
 			tokenizer.pauseBackgroundTokenize();
 			requestAnimationFrame(() => {
-				tokenizer.resumeBackgroundTokenize();
+				if (this.#tokenizer === tokenizer) tokenizer.resumeBackgroundTokenize();
 			});
 		}
 	}
@@ -720,6 +720,10 @@ var Editor = class {
 			};
 		}
 		const textDocument = editSession.document;
+		if (this.#tokenizer != null && this.#tokenizer.highlighter !== highlighter) {
+			this.#tokenizer.cleanUp();
+			this.#tokenizer = void 0;
+		}
 		if (this.#tokenizer == null && textDocument != null) this.#tokenizer = new EditorTokenizer({
 			highlighter,
 			textDocument,