@lvce-editor/editor-worker 9.0.0 → 10.0.0

package/dist/editorWorkerMain.js

@@ -1774,53 +1774,480 @@ const setDeltaY$2 = (state, value) => {
   };
 };
 
-const splitLines = lines => {
-  if (!lines) {
-    return [''];
+const Link$1 = 'Link';
+const Function = 'Function';
+const Parameter = 'Parameter';
+const Type = 'Type';
+const VariableName = 'VariableName';
+const Class = 'Class';
+
+const Link = 1;
+const Ts2816 = 2816;
+const Ts2817 = 2817;
+const Ts2824 = 2824;
+const Ts2825 = 2825;
+const Ts2856 = 2956;
+const Ts2857 = 2857;
+const Ts3072 = 3072;
+const Ts3073 = 3073;
+const Ts3077 = 3077;
+const Ts3088 = 3088;
+const Ts1792 = 1792;
+const Ts1793 = 1793;
+const Ts512 = 512;
+const Ts513 = 513;
+const Ts769 = 769;
+const Ts1024 = 1024;
+const Ts1536 = 1536;
+const Ts1537 = 1537;
+const Ts1544 = 1544;
+const Ts1545 = 1545;
+const Ts2048 = 2048;
+const Ts2049 = 2049;
+const Ts2056 = 2056;
+const Ts2057 = 2057;
+const Ts2064 = 2064;
+const Ts2080 = 2080;
+const Ts2081 = 2081;
+const Ts2088 = 2088;
+const Ts2089 = 2089;
+const Ts2313 = 2313;
+const Ts2560 = 2560;
+const Ts2561 = 2561;
+const Ts2569 = 2569;
+const Ts2584 = 2584;
+const Ts256 = 256;
+const Ts257 = 257;
+const Ts272 = 272;
+
+const getDecorationClassName = type => {
+  switch (type) {
+    case Link:
+      return Link$1;
+    case Ts2816:
+    case Ts2817:
+    case Ts2824:
+    case Ts2825:
+    case Ts2856:
+    case Ts2857:
+    case Ts3072:
+    case Ts3073:
+    case Ts3077:
+    case Ts3088:
+      return Function;
+    case Ts1792:
+    case Ts1793:
+      return Parameter;
+    case Ts512:
+    case Ts513:
+    case Ts769:
+    case Ts1024:
+    case Ts1536:
+    case Ts1537:
+    case Ts1544:
+    case Ts1545:
+      return Type;
+    case Ts2048:
+    case Ts2049:
+    case Ts2056:
+    case Ts2057:
+    case Ts2064:
+    case Ts2080:
+    case Ts2081:
+    case Ts2088:
+    case Ts2089:
+    case Ts2313:
+    case Ts2560:
+    case Ts2561:
+    case Ts2569:
+    case Ts2584:
+      return VariableName;
+    case Ts256:
+    case Ts257:
+    case Ts272:
+      return Class;
+    default:
+      return `Unknown-${type}`;
   }
-  return lines.split('\n');
 };
 
-// based on https://github.com/microsoft/vscode/blob/c0769274fa136b45799edeccc0d0a2f645b75caf/src/vs/base/common/arrays.ts#L625 (License MIT)
+const deepCopy = value => {
+  return structuredClone(value);
+};
 
-// @ts-ignore
-const insertInto = (array, start, newItems) => {
-  const originalLength = array.length;
-  const newItemsLength = newItems.length;
-  array.length = originalLength + newItemsLength;
-  // Move the items after the start index, start from the end so that we don't overwrite any value.
-  for (let i = originalLength - 1; i >= start; i--) {
-    array[i + newItemsLength] = array[i];
+const getInitialLineState = initialLineState => {
+  return deepCopy(initialLineState);
+};
+
+const state$8 = {
+  warned: []
+};
+const flattenTokensArray = tokens => {
+  const flattened = [];
+  for (const token of tokens) {
+    object(token);
+    flattened.push(token.type, token.length);
   }
-  for (let i = 0; i < newItemsLength; i++) {
-    array[i + start] = newItems[i];
+  return flattened;
+};
+const warnDeprecatedArrayReturn = (languageId, fn) => {
+  if (state$8.warned.includes(fn)) {
+    return;
+  }
+  state$8.warned.push(fn);
+  console.warn(`tokenizers without hasArrayReturn=false are deprecated (language ${languageId})`);
+};
+const safeTokenizeLine = (languageId, tokenizeLine, line, lineStateAtStart, hasArrayReturn) => {
+  try {
+    const lineState = tokenizeLine(line, lineStateAtStart);
+    if (!lineState?.tokens || !lineState.state) {
+      throw new Error('invalid tokenization result');
+    }
+    if (!hasArrayReturn) {
+      warnDeprecatedArrayReturn(languageId, tokenizeLine);
+      // workaround for old tokenizers
+      lineState.tokens = flattenTokensArray(lineState.tokens);
+    }
+    return lineState;
+  } catch (error) {
+    console.error(error);
+    return {
+      tokens: [/* type */0, /* length */line.length],
+      lineState: lineStateAtStart
+    };
   }
 };
 
 /**
- * Alternative to the native Array.splice method, it
- * can only support limited number of items due to the maximum call stack size limit.
+ * @enum number
  */
-// @ts-ignore
-const spliceLargeArray = (array, start, deleteCount, newItems) => {
-  const result = array.splice(start, deleteCount);
-  insertInto(array, start, newItems);
-  return result;
+const State = {
+  TopLevelContent: 1
 };
 
-const joinLines = lines => {
-  return lines.join('\n');
+/**
+ * @enum number
+ */
+const TokenType = {
+  Text: 1
+};
+const TokenMap = {
+  [TokenType.Text]: 'Text'
+};
+const initialLineState = {
+  state: State.TopLevelContent
+};
+const hasArrayReturn = true;
+const tokenizeLine = (line, lineState) => {
+  return {
+    tokens: [TokenType.Text, line.length],
+    state: lineState.state
+  };
 };
 
-const RE_WHITESPACE = /^\s+/;
+const TokenizePlainText = {
+  __proto__: null,
+  State,
+  TokenMap,
+  TokenType,
+  hasArrayReturn,
+  initialLineState,
+  tokenizeLine
+};
 
-// TODO this doesn't belong here
-const getIndent = line => {
-  const whitespaceMatch = line.match(RE_WHITESPACE);
-  if (!whitespaceMatch) {
-    return '';
+let enabled$1 = false;
+const setEnabled$1 = value => {
+  enabled$1 = value;
+};
+const getEnabled$1 = () => {
+  return enabled$1;
+};
+
+const {
+  set: set$5,
+  invoke: invoke$7} = SyntaxHighlightingWorker;
+
+const state$7 = {
+  tokenizers: Object.create(null),
+  pending: Object.create(null)};
+const has = languageId => {
+  return languageId in state$7.tokenizers;
+};
+const set$4 = (languageId, tokenizer) => {
+  state$7.tokenizers[languageId] = tokenizer;
+};
+const get$3 = languageId => {
+  return state$7.tokenizers[languageId];
+};
+const isPending = languageId => {
+  return languageId in state$7.pending;
+};
+
+const tokenMaps = Object.create(null);
+const set$3 = (languageId, tokenMap) => {
+  tokenMaps[languageId] = tokenMap;
+};
+const get$2 = languageId => {
+  return tokenMaps[languageId] || {};
+};
+
+// TODO loadTokenizer should be invoked from renderer worker
+const loadTokenizer = async (languageId, tokenizePath) => {
+  if (!tokenizePath) {
+    return;
   }
-  return whitespaceMatch[0];
+  if (getEnabled$1()) {
+    // @ts-ignore
+    const tokenMap = await invoke$7('Tokenizer.load', languageId, tokenizePath);
+    set$3(languageId, tokenMap);
+    return;
+  }
+  try {
+    // TODO check that tokenizer is valid
+    // 1. tokenizeLine should be of type function
+    // 2. getTokenClass should be of type function
+    const tokenizer = await import(tokenizePath);
+    if (typeof tokenizer.tokenizeLine !== 'function') {
+      console.warn(`tokenizer.tokenizeLine should be a function in "${tokenizePath}"`);
+      return;
+    }
+    if (!tokenizer.TokenMap || typeof tokenizer.TokenMap !== 'object' || Array.isArray(tokenizer.TokenMap)) {
+      console.warn(`tokenizer.TokenMap should be an object in "${tokenizePath}"`);
+      return;
+    }
+    set$3(languageId, tokenizer.TokenMap);
+    set$4(languageId, tokenizer);
+  } catch (error) {
+    // TODO better error handling
+    console.error(error);
+  }
+};
+const getTokenizer = languageId => {
+  if (has(languageId)) {
+    return get$3(languageId);
+  }
+  if (isPending(languageId)) {
+    return TokenizePlainText;
+  }
+  return TokenizePlainText;
+};
+
+const tokenizers = Object.create(null);
+const set$2 = (id, value) => {
+  tokenizers[id] = value;
+};
+const get$1 = id => {
+  return tokenizers[id] || TokenizePlainText;
+};
+
+const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
+  const tokenizersToLoad = [];
+  const embeddedResults = [];
+  let topContext;
+  for (const index of linesWithEmbed) {
+    const result = lineCache[index + 1];
+    const line = lines[index];
+    if (result.embeddedLanguage) {
+      const {
+        embeddedLanguage,
+        embeddedLanguageStart,
+        embeddedLanguageEnd
+      } = result;
+      const embeddedTokenizer = getTokenizer(embeddedLanguage);
+      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
+        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
+        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
+        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
+        topContext = embedResult;
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push({
+          result: embedResult,
+          TokenMap: embeddedTokenizer.TokenMap,
+          isFull
+        });
+      } else if (line.length === 0) {
+        const embedResult = {
+          tokens: []
+        };
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push({
+          result: embedResult,
+          isFull: true,
+          TokenMap: []
+        });
+      } else {
+        tokenizersToLoad.push(embeddedLanguage);
+        embeddedResults.push({
+          result: {},
+          isFull: false,
+          TokenMap: []
+        });
+        topContext = undefined;
+      }
+    } else {
+      topContext = undefined;
+    }
+  }
+  return {
+    tokenizersToLoad,
+    embeddedResults
+  };
+};
+const getTokenizeEndIndex = (invalidStartIndex, endLineIndex, tokenizeStartIndex) => {
+  return invalidStartIndex < endLineIndex ? endLineIndex : tokenizeStartIndex;
+};
+
+// TODO only send changed lines to renderer process instead of all lines in viewport
+const getTokensViewport = (editor, startLineIndex, endLineIndex) => {
+  const {
+    invalidStartIndex,
+    lineCache,
+    tokenizerId,
+    lines,
+    languageId
+  } = editor;
+  const tokenizer = get$1(tokenizerId);
+  const {
+    hasArrayReturn,
+    tokenizeLine,
+    initialLineState
+  } = tokenizer;
+  const tokenizeStartIndex = invalidStartIndex;
+  const tokenizeEndIndex = getTokenizeEndIndex(invalidStartIndex, endLineIndex, tokenizeStartIndex);
+  const tokenizersToLoad = [];
+  const embeddedResults = [];
+  const linesWithEmbed = [];
+  for (let i = tokenizeStartIndex; i < tokenizeEndIndex; i++) {
+    const lineState = i === 0 ? getInitialLineState(initialLineState) : lineCache[i];
+    const line = lines[i];
+    const result = safeTokenizeLine(languageId, tokenizeLine, line, lineState, hasArrayReturn);
+    // TODO if lineCacheEnd matches the one before, skip tokenizing lines after
+    lineCache[i + 1] = result;
+    if (result.embeddedLanguage) {
+      result.embeddedResultIndex = linesWithEmbed.length;
+      linesWithEmbed.push(i);
+    }
+  }
+  const visibleLines = lineCache.slice(startLineIndex + 1, endLineIndex + 1);
+  if (linesWithEmbed.length > 0) {
+    const {
+      tokenizersToLoad,
+      embeddedResults
+    } = getTokensViewportEmbedded(languageId, lines, lineCache, linesWithEmbed);
+    // TODO support lineCache with embedded content
+    editor.invalidStartIndex = 0;
+    return {
+      tokens: visibleLines,
+      tokenizersToLoad,
+      embeddedResults
+    };
+  }
+  editor.invalidStartIndex = Math.max(invalidStartIndex, tokenizeEndIndex);
+  return {
+    tokens: visibleLines,
+    tokenizersToLoad,
+    embeddedResults
+  };
+};
+
+const sentLines = Object.create(null);
+
+// TODO only send changed lines to renderer process instead of all lines in viewport
+const getTokensViewport2 = async (editor, startLineIndex, endLineIndex, syncIncremental) => {
+  if (getEnabled$1()) {
+    if (syncIncremental) {
+      const {
+        invalidStartIndex,
+        lines,
+        languageId,
+        id
+      } = editor;
+      let hasLinesToSend = true;
+      let linesToSend = lines;
+      if (sentLines[id] === lines) {
+        hasLinesToSend = false;
+        linesToSend = [];
+      } else {
+        sentLines[id] = lines;
+      }
+      const slimEditor = {
+        languageId,
+        invalidStartIndex
+      };
+      return invoke$7('GetTokensViewport.getTokensViewport', slimEditor,
+      // @ts-ignore
+      startLineIndex, endLineIndex, hasLinesToSend, id, linesToSend);
+    }
+    // TODO only send needed lines of text
+    // @ts-ignore
+    return invoke$7('GetTokensViewport.getTokensViewport', editor, startLineIndex, endLineIndex, true, editor.id, editor.lines);
+  }
+  return getTokensViewport(editor, startLineIndex, endLineIndex);
+};
+
+const loadTokenizers = async languageIds => {
+  for (const languageId of languageIds) {
+    // @ts-ignore
+    await loadTokenizer(languageId);
+  }
+};
+
+const EmptyString = '';
+const NewLine = '\n';
+const Space$1 = ' ';
+const Tab$1 = '\t';
+const DoubleQuote$1 = '"';
+
+const normalizeText = (text, normalize, tabSize) => {
+  if (normalize) {
+    return text.replaceAll(Tab$1, Space$1.repeat(tabSize));
+  }
+  return text;
+};
+const shouldNormalizeText = text => {
+  return text.includes(Tab$1);
+};
+
+// based on https://github.com/microsoft/vscode/blob/c0769274fa136b45799edeccc0d0a2f645b75caf/src/vs/base/common/arrays.ts#L625 (License MIT)
+
+// @ts-ignore
+const insertInto = (array, start, newItems) => {
+  const originalLength = array.length;
+  const newItemsLength = newItems.length;
+  array.length = originalLength + newItemsLength;
+  // Move the items after the start index, start from the end so that we don't overwrite any value.
+  for (let i = originalLength - 1; i >= start; i--) {
+    array[i + newItemsLength] = array[i];
+  }
+  for (let i = 0; i < newItemsLength; i++) {
+    array[i + start] = newItems[i];
+  }
+};
+
+/**
+ * Alternative to the native Array.splice method, it
+ * can only support limited number of items due to the maximum call stack size limit.
+ */
+// @ts-ignore
+const spliceLargeArray = (array, start, deleteCount, newItems) => {
+  const result = array.splice(start, deleteCount);
+  insertInto(array, start, newItems);
+  return result;
+};
+
+const joinLines = lines => {
+  return lines.join('\n');
+};
+
+const RE_WHITESPACE = /^\s+/;
+
+// TODO this doesn't belong here
+const getIndent = line => {
+  const whitespaceMatch = line.match(RE_WHITESPACE);
+  if (!whitespaceMatch) {
+    return '';
+  }
+  return whitespaceMatch[0];
 };
 
 // TODO have function for single edit (most common, avoid one array)
@@ -1966,114 +2393,385 @@ const positionAt = (textDocument, offset) => {
   };
 };
 
-const getSelectionPairs = (selections, i) => {
-  const first = selections[i];
-  const second = selections[i + 1];
-  const third = selections[i + 2];
-  const fourth = selections[i + 3];
-  if (first > third || first === third && second >= fourth) {
-    return [third, fourth, first, second, 1];
-  }
-  return [first, second, third, fourth, 0];
-};
-
-const EmptyString = '';
-const NewLine = '\n';
-const Space$1 = ' ';
-const Tab$1 = '\t';
-const DoubleQuote$1 = '"';
+// const getTokensIncremental = (editor, min, max) => {
+//   const currentLength = editor.lineStateCache.length
+//   const tokens = []
+//   const lines = editor.lines
+//   let lineState = editor.tokenizer.initialLineState
+//   for (let i = currentLength; i < max; i++) {
+//     const line = lines[i]
+//     try {
+//       lineState = editor.tokenizer.tokenizeLine(line, lineState)
+//       if (!lineState || !lineState.tokens || !lineState.state) {
+//         throw new Error('invalid tokenization result')
+//       }
+//     } catch (error) {
+//       tokens.push([{ length: line.length, type: 0 }])
+//       console.error(error)
+//       // renderWithoutSyntaxHighlighting(state, firstRow, lastRow)
+//       continue
+//     }
+//     const newTokens = lineState.tokens
+//     tokens.push(newTokens)
+//   }
+//   return tokens
+// }
 
-const getTabCount = string => {
-  let count = 0;
-  for (const element of string) {
-    if (element === Tab$1) {
-      count++;
+// const getLineInfosIncremental = (editor, tokens, minLineY, maxLineY) => {
+//   const result = []
+//   const lines = editor.lines
+//   const TokenMap = editor.tokenizer.TokenMap
+//   for (let i = minLineY; i < maxLineY; i++) {
+//     result.push(getLineInfo(lines[i], tokens[i], TokenMap))
+//   }
+//   return result
+// }
+
+const getStartDefaults = (tokens, minOffset) => {
+  let start = 0;
+  let end = 0;
+  let startIndex = 0;
+  const tokensLength = tokens.length;
+  for (let i = 0; i < tokensLength; i += 2) {
+    const tokenLength = tokens[i + 1];
+    end += tokenLength;
+    start = end;
+    if (start >= minOffset) {
+      start -= tokenLength;
+      end -= tokenLength;
+      startIndex = i;
+      break;
     }
   }
-  return count;
-};
-
-const measureTextWidthFast = (text, charWidth) => {
-  return text.length * charWidth;
-};
-
-const getFontString = (fontWeight, fontSize, fontFamily) => {
-  return `${fontWeight} ${fontSize}px ${fontFamily}`;
-};
-
-const getLetterSpacingString = letterSpacing => {
-  return `${letterSpacing}px`;
+  return {
+    start,
+    end,
+    startIndex
+  };
 };
-
-const createMeasureContext = () => {
-  const canvas = new OffscreenCanvas(0, 0);
-  const ctx = /** @type {OffscreenCanvasRenderingContext2D} */canvas.getContext('2d');
-  if (!ctx) {
-    throw new Error('Failed to get canvas context 2d');
+const getLineInfoEmbeddedFull = (embeddedResults, tokenResults, line, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset) => {
+  const lineInfo = [];
+  const embeddedResult = embeddedResults[tokenResults.embeddedResultIndex];
+  const embeddedTokens = embeddedResult.result.tokens;
+  const embeddedTokenMap = embeddedResult.TokenMap;
+  const tokensLength = embeddedTokens.length;
+  let {
+    startIndex,
+    start,
+    end
+  } = getStartDefaults(embeddedTokens, minOffset);
+  const difference = getDifference(start, averageCharWidth, deltaX);
+  for (let i = startIndex; i < tokensLength; i += 2) {
+    const tokenType = embeddedTokens[i];
+    const tokenLength = embeddedTokens[i + 1];
+    end += tokenLength;
+    const className = `Token ${embeddedTokenMap[tokenType] || 'Unknown'}`;
+    const text = line.slice(start, end);
+    const normalizedText = normalizeText(text, normalize, tabSize);
+    lineInfo.push(normalizedText, className);
+    start = end;
+    if (end >= maxOffset) {
+      break;
+    }
   }
-  return ctx;
-};
-
-const state$8 = {
-  ctx: undefined
+  return {
+    lineInfo,
+    difference
+  };
 };
-const getOrCreate$4 = createCtx => {
-  if (state$8.ctx) {
-    return state$8.ctx;
+const getOffsets = (deltaX, width, averageCharWidth) => {
+  // TODO accurately measure char widths using offscreen canvas
+  // and use fast measurements for monospace ascii text
+  if (deltaX === 0) {
+    return {
+      minOffset: 0,
+      maxOffset: Math.ceil(width / averageCharWidth)
+    };
   }
-  state$8.ctx = createCtx();
-  return state$8.ctx;
+  const minOffset = Math.ceil(deltaX / averageCharWidth);
+  const maxOffset = minOffset + Math.ceil(width / averageCharWidth);
+  return {
+    minOffset,
+    maxOffset
+  };
 };
-
-const getContext = () => {
-  const ctx = getOrCreate$4(createMeasureContext);
-  return ctx;
+const getDifference = (start, averageCharWidth, deltaX) => {
+  const beforeWidth = start * averageCharWidth;
+  const difference = beforeWidth - deltaX;
+  return difference;
 };
-
-// TODO for text editor, could dispose measuring canvas after editor has been initialized to free up offscreencanvas space
-
-const measureTextWidthSlow = (text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth) => {
-  string(text);
-  number(fontWeight);
-  number(fontSize);
-  string(fontFamily);
-  boolean(isMonoSpaceFont);
-  number(charWidth);
-  if (typeof letterSpacing !== 'number') {
-    throw new TypeError('letterSpacing must be of type number');
+const getLineInfoDefault = (line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset) => {
+  const lineInfo = [];
+  let decorationIndex = 0;
+  for (; decorationIndex < decorations.length; decorationIndex += 3) {
+    const decorationOffset = decorations[decorationIndex];
+    if (decorationOffset >= lineOffset) {
+      break;
+    }
   }
-  const letterSpacingString = getLetterSpacingString(letterSpacing);
-  const fontString = getFontString(fontWeight, fontSize, fontFamily);
-  const ctx = getContext();
-  ctx.letterSpacing = letterSpacingString;
-  ctx.font = fontString;
-  const metrics = ctx.measureText(text);
   const {
-    width
-  } = metrics;
-  return width;
+    tokens
+  } = tokenResults;
+  let {
+    startIndex,
+    start,
+    end
+  } = getStartDefaults(tokens, minOffset);
+  const difference = getDifference(start, averageCharWidth, deltaX);
+  const tokensLength = tokens.length;
+  for (let i = startIndex; i < tokensLength; i += 2) {
+    const tokenType = tokens[i];
+    const tokenLength = tokens[i + 1];
+    const decorationOffset = decorations[decorationIndex];
+    let extraClassName = '';
+    if (decorationOffset !== undefined && decorationOffset - lineOffset === start) {
+      // @ts-ignore
+      decorations[++decorationIndex];
+      const decorationType = decorations[++decorationIndex];
+      // @ts-ignore
+      decorations[++decorationIndex];
+      //   decorationIndex,
+      //   decorationLength,
+      //   decorationType,
+      //   decorationModifiers,
+      // })
+      extraClassName = getDecorationClassName(decorationType);
+    }
+    end += tokenLength;
+    const text = line.slice(start, end);
+    const className = `Token ${extraClassName || TokenMap[tokenType] || 'Unknown'}`;
+    const normalizedText = normalizeText(text, normalize, tabSize);
+    lineInfo.push(normalizedText, className);
+    start = end;
+    if (end >= maxOffset) {
+      break;
+    }
+  }
+  return {
+    lineInfo,
+    difference
+  };
 };
-
-const measureTextWidth = (text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth) => {
-  if (isMonoSpaceFont) {
-    return measureTextWidthFast(text, charWidth);
+const getLineInfo$1 = (line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth) => {
+  const {
+    minOffset,
+    maxOffset
+  } = getOffsets(deltaX, width, averageCharWidth);
+  if (embeddedResults.length > 0 && tokenResults.embeddedResultIndex !== undefined) {
+    const embeddedResult = embeddedResults[tokenResults.embeddedResultIndex];
+    if (embeddedResult?.isFull) {
+      return getLineInfoEmbeddedFull(embeddedResults, tokenResults, line, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset);
+    }
   }
-  return measureTextWidthSlow(text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth);
+  return getLineInfoDefault(line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset);
 };
 
-const normalizeText = (text, normalize, tabSize) => {
-  if (normalize) {
-    return text.replaceAll(Tab$1, Space$1.repeat(tabSize));
+// TODO need lots of tests for this
+const getLineInfosViewport = (editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth) => {
+  const result = [];
+  const differences = [];
+  const {
+    lines,
+    decorations,
+    languageId
+  } = editor;
+  const tokenMap = get$2(languageId);
+  let offset = minLineOffset;
+  const tabSize = 2;
+  for (let i = minLineY; i < maxLineY; i++) {
+    const line = lines[i];
+    const normalize = shouldNormalizeText(line);
+    const {
+      lineInfo,
+      difference
+    } = getLineInfo$1(line, tokens[i - minLineY], embeddedResults, decorations, tokenMap, offset, normalize, tabSize, width, deltaX, averageCharWidth);
+    result.push(lineInfo);
+    differences.push(difference);
+    offset += line.length + 1;
   }
-  return text;
+  return {
+    result,
+    differences
+  };
 };
-const shouldNormalizeText = text => {
-  return text.includes(Tab$1);
+const getVisible$1 = async (editor, syncIncremental) => {
+  // TODO should separate rendering from business logic somehow
+  // currently hard to test because need to mock editor height, top, left,
+  // invalidStartIndex, lineCache, etc. just for testing editorType
+  // editor.invalidStartIndex = changes[0].start.rowIndex
+  // @ts-ignore
+  const {
+    minLineY,
+    numberOfVisibleLines,
+    lines,
+    width,
+    deltaX,
+    charWidth
+  } = editor;
+  const maxLineY = Math.min(minLineY + numberOfVisibleLines, lines.length);
+  // @ts-ignore
+  const {
+    tokens,
+    tokenizersToLoad,
+    embeddedResults
+  } = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
+  const minLineOffset = offsetAtSync(editor, minLineY, 0);
+  const averageCharWidth = charWidth;
+  const {
+    result,
+    differences
+  } = getLineInfosViewport(editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth);
+  if (tokenizersToLoad.length > 0) {
+    loadTokenizers(tokenizersToLoad);
+  }
+  return {
+    textInfos: result,
+    differences
+  };
 };
 
-const getX = (line, column, fontWeight, fontSize, fontFamily, isMonospaceFont, letterSpacing, tabSize, halfCursorWidth, width, averageCharWidth, difference = 0) => {
-  if (!line) {
-    return 0;
+const emptyIncrementalEdits = [];
+
+const getIncrementalEdits = async (oldState, newState) => {
+  if (!newState.undoStack) {
+    return emptyIncrementalEdits;
+  }
+  if (oldState.undoStack === newState.undoStack) {
+    return emptyIncrementalEdits;
+  }
+  const lastChanges = newState.undoStack.at(-1);
+  if (lastChanges && lastChanges.length === 1) {
+    const lastChange = lastChanges[0];
+    if (lastChange.origin === EditorType) {
+      const {
+        rowIndex
+      } = lastChange.start;
+      const {
+        lines
+      } = newState;
+      const oldLine = oldState.lines[rowIndex];
+      const newLine = lines[rowIndex];
+      // @ts-ignore
+      const incrementalEdits = await invoke$7(
+      // @ts-ignore
+      'TokenizeIncremental.tokenizeIncremental', newState.uid,
+      // @ts-ignore
+      newState.languageId, oldLine, newLine, rowIndex, newState.minLineY);
+      if (incrementalEdits && incrementalEdits.length === 1) {
+        return incrementalEdits;
+      }
+    }
+  }
+  return emptyIncrementalEdits;
+};
+
+const splitLines = lines => {
+  if (!lines) {
+    return [''];
+  }
+  return lines.split('\n');
+};
+
+let enabled = false;
+const setEnabled = value => {
+  enabled = value;
+};
+const getEnabled = () => {
+  return enabled;
+};
+
+const getSelectionPairs = (selections, i) => {
+  const first = selections[i];
+  const second = selections[i + 1];
+  const third = selections[i + 2];
+  const fourth = selections[i + 3];
+  if (first > third || first === third && second >= fourth) {
+    return [third, fourth, first, second, 1];
+  }
+  return [first, second, third, fourth, 0];
+};
+
+const getTabCount = string => {
+  let count = 0;
+  for (const element of string) {
+    if (element === Tab$1) {
+      count++;
+    }
+  }
+  return count;
+};
+
+const measureTextWidthFast = (text, charWidth) => {
+  return text.length * charWidth;
+};
+
+const getFontString = (fontWeight, fontSize, fontFamily) => {
+  return `${fontWeight} ${fontSize}px ${fontFamily}`;
+};
+
+const getLetterSpacingString = letterSpacing => {
+  return `${letterSpacing}px`;
+};
+
+const createMeasureContext = () => {
+  const canvas = new OffscreenCanvas(0, 0);
+  const ctx = /** @type {OffscreenCanvasRenderingContext2D} */canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get canvas context 2d');
+  }
+  return ctx;
+};
+
+const state$6 = {
+  ctx: undefined
+};
+const getOrCreate$4 = createCtx => {
+  if (state$6.ctx) {
+    return state$6.ctx;
+  }
+  state$6.ctx = createCtx();
+  return state$6.ctx;
+};
+
+const getContext = () => {
+  const ctx = getOrCreate$4(createMeasureContext);
+  return ctx;
+};
+
+// TODO for text editor, could dispose measuring canvas after editor has been initialized to free up offscreencanvas space
+
+const measureTextWidthSlow = (text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth) => {
+  string(text);
+  number(fontWeight);
+  number(fontSize);
+  string(fontFamily);
+  boolean(isMonoSpaceFont);
+  number(charWidth);
+  if (typeof letterSpacing !== 'number') {
+    throw new TypeError('letterSpacing must be of type number');
+  }
+  const letterSpacingString = getLetterSpacingString(letterSpacing);
+  const fontString = getFontString(fontWeight, fontSize, fontFamily);
+  const ctx = getContext();
+  ctx.letterSpacing = letterSpacingString;
+  ctx.font = fontString;
+  const metrics = ctx.measureText(text);
+  const {
+    width
+  } = metrics;
+  return width;
+};
+
+const measureTextWidth = (text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth) => {
+  if (isMonoSpaceFont) {
+    return measureTextWidthFast(text, charWidth);
+  }
+  return measureTextWidthSlow(text, fontWeight, fontSize, fontFamily, letterSpacing, isMonoSpaceFont, charWidth);
+};
+
+const getX = (line, column, fontWeight, fontSize, fontFamily, isMonospaceFont, letterSpacing, tabSize, halfCursorWidth, width, averageCharWidth, difference = 0) => {
+  if (!line) {
+    return 0;
   }
   string(line);
   number(tabSize);
@@ -2210,7 +2908,7 @@ const getSelectionArray = visibleSelections => {
   }
   return selectionsArray;
 };
-const getVisible$1 = editor => {
+const getVisible = editor => {
   const visibleCursors = [];
   const visibleSelections = [];
   // // TODO binary search
@@ -2428,15 +3126,29 @@ const scheduleDocumentAndCursorsSelections = async (editor, changes, selectionCh
     autoClosingRanges
   };
   set$6(editor.uid, editor, newEditor);
+  const incrementalEdits = await getIncrementalEdits(editor, newEditor);
   const newWidgets = await applyWidgetChanges(newEditor, changes);
   const newEditor2 = {
     ...newEditor,
-    widgets: newWidgets
+    widgets: newWidgets,
+    incrementalEdits
+  };
+  if (incrementalEdits !== emptyIncrementalEdits) {
+    return newEditor2;
+  }
+  const syncIncremental = getEnabled();
+  const {
+    textInfos,
+    differences
+  } = await getVisible$1(newEditor2, syncIncremental);
+  return {
+    ...newEditor2,
+    textInfos,
+    differences
   };
-  return newEditor2;
 };
 // @ts-ignore
-const scheduleDocumentAndCursorsSelectionIsUndo = (editor, changes) => {
+const scheduleDocumentAndCursorsSelectionIsUndo = async (editor, changes) => {
   object(editor);
   array(changes);
   if (changes.length === 0) {
@@ -2456,7 +3168,26 @@ const scheduleDocumentAndCursorsSelectionIsUndo = (editor, changes) => {
     // undoStack: [...editor.undoStack.slice(0, -2)],
     invalidStartIndex
   };
-  return newEditor;
+  const incrementalEdits = await getIncrementalEdits(editor, newEditor);
+
+  // TODO change event should be emitted after rendering
+  const finalEditor = {
+    ...newEditor,
+    incrementalEdits
+  };
+  if (incrementalEdits !== emptyIncrementalEdits) {
+    return finalEditor;
+  }
+  const syncIncremental = getEnabled();
+  const {
+    textInfos,
+    differences
+  } = await getVisible$1(finalEditor, syncIncremental);
+  return {
+    ...finalEditor,
+    textInfos,
+    differences
+  };
 };
 
 // @ts-ignore
@@ -2481,8 +3212,26 @@ const scheduleDocument = async (editor, changes) => {
     lines: newLines,
     invalidStartIndex
   };
+  const incrementalEdits = await getIncrementalEdits(editor, newEditor);
+
   // TODO change event should be emitted after rendering
-  return newEditor;
+  const finalEditor = {
+    ...newEditor,
+    incrementalEdits
+  };
+  if (incrementalEdits !== emptyIncrementalEdits) {
+    return finalEditor;
+  }
+  const syncIncremental = getEnabled();
+  const {
+    textInfos,
+    differences
+  } = await getVisible$1(finalEditor, syncIncremental);
+  return {
+    ...finalEditor,
+    textInfos,
+    differences
+  };
   // RendererProcess.send([
   //   /* Viewlet.invoke */ 'Viewlet.send',
   //   /* id */ 'EditorText',
@@ -2548,8 +3297,8 @@ const TabCompletionExecuteTabCompletionProvider = 'ExtensionHost.executeTabCompl
 const TextDocumentSyncFull = 'ExtensionHostTextDocument.syncFull';
 
 const {
-  set: set$5,
-  invoke: invoke$7} = ExtensionHost;
+  set: set$1,
+  invoke: invoke$6} = ExtensionHost;
 
 const ColorPicker = 41;
 const EditorCompletion = 9;
@@ -2578,7 +3327,7 @@ const execute = async ({
 }) => {
   const fullEvent = `${event}:${editor.languageId}`;
   await activateByEvent(fullEvent);
-  const result = await invoke$7(method, editor.uid, ...args);
+  const result = await invoke$6(method, editor.uid, ...args);
   return result;
 };
 
@@ -2646,7 +3395,7 @@ const updateDiagnostics = async newState => {
     // TODO don't really need text document sync response
     // could perhaps save a lot of messages by using send instead of invoke
     // @ts-ignore
-    await invoke$7(TextDocumentSyncFull, newState.uri, newState.id, newState.languageId, content);
+    await invoke$6(TextDocumentSyncFull, newState.uri, newState.id, newState.languageId, content);
     const diagnostics = await executeDiagnosticProvider(newState);
     const latest = get$4(newState.id);
     if (!latest) {
@@ -2672,6 +3421,8 @@ const updateDiagnostics = async newState => {
 };
 
 const emptyEditor = {
+  textInfos: [],
+  differences: [],
   uri: '',
   languageId: '',
   // TODO use numeric language id?
@@ -2695,7 +3446,8 @@ const emptyEditor = {
   selections: new Uint32Array(),
   diagnostics: [],
   highlightedLine: -1,
-  debugEnabled: false
+  debugEnabled: false,
+  incrementalEdits: emptyIncrementalEdits
 };
 const createEditor = async ({
   id,
@@ -2732,6 +3484,7 @@ const createEditor = async ({
   string(content);
   const charWidth = measureCharacterWidth(fontWeight, fontSize, fontFamily, letterSpacing);
   const editor = {
+    textInfos: [],
     uri,
     isAutoClosingBracketsEnabled,
     isAutoClosingTagsEnabled,
@@ -2787,7 +3540,8 @@ const createEditor = async ({
     id,
     widgets: [],
     focusKey: Empty,
-    diagnosticsEnabled
+    diagnosticsEnabled,
+    incrementalEdits: emptyIncrementalEdits
   };
   // TODO avoid creating intermediate editors here
   const newEditor1 = setBounds(editor, x, y, width, height, 9);
@@ -2800,13 +3554,23 @@ const createEditor = async ({
   } else {
     newEditor3 = setDeltaY$2(newEditor2, 0);
   }
+  const syncIncremental = getEnabled();
+  const {
+    textInfos,
+    differences
+  } = await getVisible$1(newEditor3, syncIncremental);
   const newEditor4 = {
     ...newEditor3,
-    focused: true
+    focused: true,
+    textInfos,
+    differences
   };
   set$6(id, emptyEditor, newEditor4);
+
+  // TODO only sync when needed
+  // e.g. it might not always be necessary to send text to extension host worker
   // @ts-ignore
-  await invoke$7(TextDocumentSyncFull, uri, id, languageId, content);
+  await invoke$6(TextDocumentSyncFull, uri, id, languageId, content);
   if (diagnosticsEnabled) {
     updateDiagnostics(newEditor4);
   }
@@ -3180,7 +3944,7 @@ const y = (editor, rowIndex) => {
   return offsetY;
 };
 
-const state$7 = {
+const state$5 = {
   timeout: -1
 };
 
@@ -3211,7 +3975,7 @@ const editorShowMessage = async (editor, rowIndex, columnIndex, message, isError
 
     // TODO use wrapper timing module instead of this
     // @ts-ignore
-    state$7.timeout = setTimeout(handleTimeout, 3000);
+    state$5.timeout = setTimeout(handleTimeout, 3000);
   }
   return editor;
 };
@@ -3231,8 +3995,8 @@ const showErrorMessage = async (editor, rowIndex, columnIndex, message) => {
 
 // @ts-ignore
 const editorHideMessage = async editor => {
-  clearTimeout(state$7.timeout);
-  state$7.timeout = -1;
+  clearTimeout(state$5.timeout);
+  state$5.timeout = -1;
   // await RendererProcess.invoke(/* Viewlet.send */ 'Viewlet.send', /* id */ editor.uid, /* method */ 'hideOverlayMessage')
   return editor;
 };
@@ -3357,7 +4121,7 @@ const getOrCreate$3 = () => {
   }
   return workerPromise$3;
 };
-const invoke$6 = async (method, ...params) => {
+const invoke$5 = async (method, ...params) => {
   const worker = await getOrCreate$3();
   return await worker.invoke(method, ...params);
 };
@@ -3376,7 +4140,7 @@ const closeRename = async editor => {
     return editor;
   }
   const renameWidget = widgets[renameWidgetIndex];
-  await invoke$6('Rename.close', renameWidget.newState.uid);
+  await invoke$5('Rename.close', renameWidget.newState.uid);
   const latest = get$4(uid);
   const {
     newState
@@ -3495,12 +4259,12 @@ const openColorPicker = async editor => {
   return addWidgetToEditor(ColorPicker$1, ColorPicker, editor, create$6, newStateGenerator$6);
 };
 
-const state$6 = {
+const state$4 = {
   isComposing: false,
   compositionText: ''
 };
 const compositionStart = (editor, event) => {
-  state$6.isComposing = true;
+  state$4.isComposing = true;
   return editor;
 };
 const getCompositionChanges = (selections, data) => {
@@ -3510,7 +4274,7 @@ const getCompositionChanges = (selections, data) => {
     const selectionStartColumn = selections[i + 1];
     const selectionEndRow = selections[i + 2];
     const selectionEndColumn = selections[i + 3];
-    const startColumnIndex = selectionStartColumn - state$6.compositionText.length;
+    const startColumnIndex = selectionStartColumn - state$4.compositionText.length;
     changes.push({
       start: {
         rowIndex: selectionStartRow,
@@ -3521,7 +4285,7 @@ const getCompositionChanges = (selections, data) => {
         columnIndex: selectionEndColumn
       },
       inserted: [data],
-      deleted: [state$6.compositionText],
+      deleted: [state$4.compositionText],
       origin: CompositionUpdate
     });
   }
@@ -3532,7 +4296,7 @@ const compositionUpdate = (editor, data) => {
     selections
   } = editor;
   const changes = getCompositionChanges(selections, data);
-  state$6.compositionText = data;
+  state$4.compositionText = data;
   return scheduleDocumentAndCursorsSelections(editor, changes);
 };
 const compositionEnd = (editor, data) => {
@@ -3540,8 +4304,8 @@ const compositionEnd = (editor, data) => {
     selections
   } = editor;
   const changes = getCompositionChanges(selections, data);
-  state$6.isComposing = false;
-  state$6.compositionText = '';
+  state$4.isComposing = false;
+  state$4.compositionText = '';
   return scheduleDocumentAndCursorsSelections(editor, changes);
 };
 
@@ -4616,19 +5380,19 @@ const Single = 1;
 const Double = 2;
 const Triple = 3;
 
-const state$5 = {
+const state$3 = {
   position: {
     rowIndex: 0,
     columnIndex: 0
   }
 };
 const getPosition$1 = () => {
-  return state$5.position;
+  return state$3.position;
 };
 
 // @ts-ignore
 const setPosition$1 = position => {
-  state$5.position = position;
+  state$3.position = position;
 };
 
 // TODO first change cursor position, then run go to definition
@@ -4762,20 +5526,20 @@ const handleMouseDown = (state, modifier, x, y, detail) => {
   }
 };
 
-const state$4 = {
+const state$2 = {
   timeout: -1,
   x: 0,
   y: 0,
   editor: undefined
 };
-const get$3 = () => {
-  return state$4;
+const get = () => {
+  return state$2;
 };
-const set$4 = (editor, timeout, x, y) => {
-  state$4.editor = editor;
-  state$4.timeout = timeout;
-  state$4.x = x;
-  state$4.y = y;
+const set = (editor, timeout, x, y) => {
+  state$2.editor = editor;
+  state$2.timeout = timeout;
+  state$2.x = x;
+  state$2.y = y;
 };
 
 const showHover$1 = async (editor, position) => {
@@ -4797,7 +5561,7 @@ const onHoverIdle = async () => {
     x,
     y,
     editor
-  } = get$3();
+  } = get();
   at(editor, x, y);
   await showHover$1();
 };
@@ -4808,12 +5572,12 @@ const handleMouseMove = (editor, x, y) => {
   if (!editor.hoverEnabled) {
     return editor;
   }
-  const oldState = get$3();
+  const oldState = get();
   if (oldState.timeout !== -1) {
     clearTimeout(oldState.timeout);
   }
   const timeout = setTimeout(onHoverIdle, hoverDelay);
-  set$4(editor, timeout, x, y);
+  set(editor, timeout, x, y);
   return editor;
 };
 
@@ -4920,7 +5684,7 @@ const editorHandleNativeSelectionChange = (editor, range) => {
   return scheduleSelections(editor, selections);
 };
 
-const state$3 = {
+const state$1 = {
   /**
    * @type {any}
    */
@@ -4934,26 +5698,26 @@ const state$3 = {
 
 // @ts-ignore
 const setEditor = editor => {
-  state$3.currentEditor = editor;
-  state$3.hasListener = true;
+  state$1.currentEditor = editor;
+  state$1.hasListener = true;
 };
 const clearEditor = () => {
-  state$3.currentEditor = undefined;
-  state$3.hasListener = false;
+  state$1.currentEditor = undefined;
+  state$1.hasListener = false;
 };
 
 // @ts-ignore
 const setPosition = position => {
-  state$3.position = position;
+  state$1.position = position;
 };
 const getEditor$1 = () => {
-  return state$3.currentEditor;
+  return state$1.currentEditor;
 };
 const getPosition = () => {
-  return state$3.position;
+  return state$1.position;
 };
 const hasListener = () => {
-  return state$3.hasListener;
+  return state$1.hasListener;
 };
 
 // @ts-ignore
@@ -5136,7 +5900,7 @@ const handleTouchEnd = (editor, touchEvent) => {
   // }
 };
 
-const state$2 = {
+const state = {
   touchOffsetY: 0,
   deltaY: 0
 };
@@ -5147,8 +5911,8 @@ const handleTouchStart = (editor, touchEvent) => {
     return;
   }
   const firstTouch = touchEvent.touches[0];
-  state$2.touchOffsetY = firstTouch.y;
-  state$2.deltaY = editor.deltaY;
+  state.touchOffsetY = firstTouch.y;
+  state.deltaY = editor.deltaY;
   // const position = EditorPosition.at(editor, firstTouch.x, firstTouch.y)
   // EditorMoveSelection.state.position = position
   // state.date = Date.now()
@@ -5188,7 +5952,7 @@ const handleTouchMove = (editor, touchEvent) => {
     return;
   }
   const firstTouch = touchEvent.touches[0];
-  const offsetY = state$2.deltaY + (state$2.touchOffsetY - firstTouch.y);
+  const offsetY = state.deltaY + (state.touchOffsetY - firstTouch.y);
   setDeltaYFixedValue(editor, offsetY);
 };
 
@@ -5624,7 +6388,7 @@ const getOrCreate$2 = () => {
   }
   return workerPromise$2;
 };
-const invoke$5 = async (method, ...params) => {
+const invoke$4 = async (method, ...params) => {
   const worker = await getOrCreate$2();
   return await worker.invoke(method, ...params);
 };
@@ -5643,10 +6407,10 @@ const newStateGenerator$4 = async (state, parentUid) => {
   const {
     languageId
   } = newState;
-  await invoke$5('Completions.create', uid, x, y, width, height, parentUid, languageId);
-  await invoke$5('Completions.loadContent', uid);
-  const diff = await invoke$5('Completions.diff2', uid);
-  const commands = await invoke$5('Completions.render2', uid, diff);
+  await invoke$4('Completions.create', uid, x, y, width, height, parentUid, languageId);
+  await invoke$4('Completions.loadContent', uid);
+  const diff = await invoke$4('Completions.diff2', uid);
+  const commands = await invoke$4('Completions.render2', uid, diff);
   return {
     ...state,
     commands
@@ -5698,7 +6462,7 @@ const launch = async () => {
   const rpc = await launchFindWidgetWorker();
   set$c(rpcId, rpc);
 };
-const invoke$4 = async (method, ...params) => {
+const invoke$3 = async (method, ...params) => {
   const rpc = get$6(rpcId);
   return await rpc.invoke(method, ...params);
 };
@@ -5737,10 +6501,10 @@ const loadContent$1 = async (state, parentUid) => {
     height
   } = editor;
   await launch();
-  await invoke$4('FindWidget.create', uid, x, y, width, height, parentUid);
-  await invoke$4('FindWidget.loadContent', uid);
-  const diff = await invoke$4('FindWidget.diff2', uid);
-  const commands = await invoke$4('FindWidget.render2', uid, diff);
+  await invoke$3('FindWidget.create', uid, x, y, width, height, parentUid);
+  await invoke$3('FindWidget.loadContent', uid);
+  const diff = await invoke$3('FindWidget.diff2', uid);
+  const commands = await invoke$3('FindWidget.render2', uid, diff);
   return {
     ...state,
     commands
@@ -5813,10 +6577,10 @@ const newStateGenerator$2 = async (state, parentUid) => {
   const {
     languageId
   } = newState;
-  await invoke$6('Rename.create', uid, x, y, width, height, parentUid, languageId);
-  await invoke$6('Rename.loadContent', uid);
-  const diff = await invoke$6('Rename.diff2', uid);
-  const commands = await invoke$6('Rename.render2', uid, diff);
+  await invoke$5('Rename.create', uid, x, y, width, height, parentUid, languageId);
+  await invoke$5('Rename.loadContent', uid);
+  const diff = await invoke$5('Rename.diff2', uid);
+  const commands = await invoke$5('Rename.render2', uid, diff);
   return {
     ...state,
     commands
@@ -6556,155 +7320,45 @@ const setDecorations = (editor, decorations, diagnostics) => {
   };
 };
 
-let enabled$1 = false;
-const setEnabled$1 = value => {
-  enabled$1 = value;
-};
-const getEnabled$1 = () => {
-  return enabled$1;
-};
-
-const {
-  set: set$3,
-  invoke: invoke$3} = SyntaxHighlightingWorker;
+// TODO add command to set language id
+// without needing to specify tokenizePath
+const setLanguageId = async (editor, languageId, tokenizePath) => {
+  const {
+    tokenizerId
+  } = editor;
+  // TODO move tokenizer to syntax highlighting worker
+  // TODO only load tokenizer if not already loaded
+  // if already loaded just set tokenizer and rerender text
+  // TODO race condition
+  await loadTokenizer(languageId, tokenizePath);
+  const tokenizer = getTokenizer(languageId);
+  const newTokenizerId = tokenizerId + 1;
+  set$2(newTokenizerId, tokenizer);
+  const latest = getEditor(editor.uid);
+  if (!latest) {
+    return editor;
+  }
+  const syncIncremental = getEnabled();
+  const {
+    textInfos,
+    differences
+  } = await getVisible$1(editor, syncIncremental);
+  const newEditor4 = {
+    ...latest,
+    focused: true,
+    textInfos,
+    differences
+  };
 
-/**
- * @enum number
- */
-const State = {
-  TopLevelContent: 1
-};
+  // TODO don't update editor if tokenizer was already loaded
+  // TODO update syntax highlighting
+  // TODO get edits
 
-/**
- * @enum number
- */
-const TokenType = {
-  Text: 1
-};
-const TokenMap = {
-  [TokenType.Text]: 'Text'
-};
-const initialLineState = {
-  state: State.TopLevelContent
-};
-const hasArrayReturn = true;
-const tokenizeLine = (line, lineState) => {
   return {
-    tokens: [TokenType.Text, line.length],
-    state: lineState.state
-  };
-};
-
-const TokenizePlainText = {
-  __proto__: null,
-  State,
-  TokenMap,
-  TokenType,
-  hasArrayReturn,
-  initialLineState,
-  tokenizeLine
-};
-
-const state$1 = {
-  tokenizers: Object.create(null),
-  pending: Object.create(null)};
-const has = languageId => {
-  return languageId in state$1.tokenizers;
-};
-const set$2 = (languageId, tokenizer) => {
-  state$1.tokenizers[languageId] = tokenizer;
-};
-const get$2 = languageId => {
-  return state$1.tokenizers[languageId];
-};
-const isPending = languageId => {
-  return languageId in state$1.pending;
-};
-
-const tokenMaps = Object.create(null);
-const set$1 = (languageId, tokenMap) => {
-  tokenMaps[languageId] = tokenMap;
-};
-const get$1 = languageId => {
-  return tokenMaps[languageId] || {};
-};
-
-// TODO loadTokenizer should be invoked from renderer worker
-const loadTokenizer = async (languageId, tokenizePath) => {
-  if (!tokenizePath) {
-    return;
-  }
-  if (getEnabled$1()) {
-    // @ts-ignore
-    const tokenMap = await invoke$3('Tokenizer.load', languageId, tokenizePath);
-    set$1(languageId, tokenMap);
-    return;
-  }
-  try {
-    // TODO check that tokenizer is valid
-    // 1. tokenizeLine should be of type function
-    // 2. getTokenClass should be of type function
-    const tokenizer = await import(tokenizePath);
-    if (typeof tokenizer.tokenizeLine !== 'function') {
-      console.warn(`tokenizer.tokenizeLine should be a function in "${tokenizePath}"`);
-      return;
-    }
-    if (!tokenizer.TokenMap || typeof tokenizer.TokenMap !== 'object' || Array.isArray(tokenizer.TokenMap)) {
-      console.warn(`tokenizer.TokenMap should be an object in "${tokenizePath}"`);
-      return;
-    }
-    set$1(languageId, tokenizer.TokenMap);
-    set$2(languageId, tokenizer);
-  } catch (error) {
-    // TODO better error handling
-    console.error(error);
-  }
-};
-const getTokenizer = languageId => {
-  if (has(languageId)) {
-    return get$2(languageId);
-  }
-  if (isPending(languageId)) {
-    return TokenizePlainText;
-  }
-  return TokenizePlainText;
-};
-
-const tokenizers = Object.create(null);
-const set = (id, value) => {
-  tokenizers[id] = value;
-};
-const get = id => {
-  return tokenizers[id] || TokenizePlainText;
-};
-
-// TODO add command to set language id
-// without needing to specify tokenizePath
-const setLanguageId = async (editor, languageId, tokenizePath) => {
-  const {
-    tokenizerId
-  } = editor;
-  // TODO move tokenizer to syntax highlighting worker
-  // TODO only load tokenizer if not already loaded
-  // if already loaded just set tokenizer and rerender text
-  // TODO race condition
-  await loadTokenizer(languageId, tokenizePath);
-  const tokenizer = getTokenizer(languageId);
-  const newTokenizerId = tokenizerId + 1;
-  set(newTokenizerId, tokenizer);
-  const latest = getEditor(editor.uid);
-  if (!latest) {
-    return editor;
-  }
-  // TODO don't update editor if tokenizer was already loaded
-  // TODO update syntax highlighting
-  // TODO get edits
-
-  return {
-    ...latest,
-    languageId,
-    invalidStartIndex: 0,
-    tokenizerId: newTokenizerId
+    ...newEditor4,
+    languageId,
+    invalidStartIndex: 0,
+    tokenizerId: newTokenizerId
   };
 };
 
@@ -7746,11 +8400,11 @@ const getWidgetInvoke = widgetId => {
     case ColorPicker$1:
       return invoke$8;
     case Completion:
-      return invoke$5;
-    case Find:
       return invoke$4;
+    case Find:
+      return invoke$3;
     case Rename$1:
-      return invoke$6;
+      return invoke$5;
     case SourceAction$1:
       return invoke$1;
     case Hover:
@@ -7953,8 +8607,9 @@ const {
   toggleReplace,
   toggleUseRegularExpression,
   focusNextElement,
-  focusPreviousElement
-} = createFns(['close', 'focusCloseButton', 'focusFind', 'focusNext', 'focusNextMatchButton', 'focusPrevious', 'focusPreviousMatchButton', 'focusReplace', 'focusReplaceAllButton', 'focusReplaceButton', 'focusToggleReplace', 'handleBlur', 'handleClickButton', 'handleFocus', 'handleInput', 'handleReplaceFocus', 'handleReplaceInput', 'handleToggleReplaceFocus', 'replace', 'replaceAll', 'toggleMatchCase', 'toggleMatchWholeWord', 'toggleReplace', 'toggleUseRegularExpression', 'focusNextElement', 'focusPreviousElement'], 'FindWidget', Find);
+  focusPreviousElement,
+  togglePreserveCase
+} = createFns(['close', 'focusCloseButton', 'focusFind', 'focusNext', 'focusNextMatchButton', 'focusPrevious', 'focusPreviousMatchButton', 'focusReplace', 'focusReplaceAllButton', 'focusReplaceButton', 'focusToggleReplace', 'handleBlur', 'handleClickButton', 'handleFocus', 'handleInput', 'handleReplaceFocus', 'handleReplaceInput', 'handleToggleReplaceFocus', 'replace', 'replaceAll', 'toggleMatchCase', 'toggleMatchWholeWord', 'toggleReplace', 'toggleUseRegularExpression', 'focusNextElement', 'focusPreviousElement', 'togglePreserveCase'], 'FindWidget', Find);
 
 const EditorFindWidget = {
   __proto__: null,
@@ -7985,6 +8640,7 @@ const EditorFindWidget = {
   replaceAll,
   toggleMatchCase,
   toggleMatchWholeWord,
+  togglePreserveCase,
   toggleReplace,
   toggleUseRegularExpression
 };
@@ -8015,15 +8671,7 @@ const measureTextBlockHeight = async (text, fontFamily, fontSize, lineHeight, wi
   return 100;
 };
 
-const deepCopy = value => {
-  return structuredClone(value);
-};
-
-const getInitialLineState = initialLineState => {
-  return deepCopy(initialLineState);
-};
-
-const getLineInfo$1 = (line, tokens, TokenMap) => {
+const getLineInfo = (line, tokens, TokenMap) => {
   const tokensLength = tokens.length;
   let end = 0;
   let start = 0;
@@ -8041,45 +8689,6 @@ const getLineInfo$1 = (line, tokens, TokenMap) => {
   return lineInfo;
 };
 
-const state = {
-  warned: []
-};
-const flattenTokensArray = tokens => {
-  const flattened = [];
-  for (const token of tokens) {
-    object(token);
-    flattened.push(token.type, token.length);
-  }
-  return flattened;
-};
-const warnDeprecatedArrayReturn = (languageId, fn) => {
-  if (state.warned.includes(fn)) {
-    return;
-  }
-  state.warned.push(fn);
-  console.warn(`tokenizers without hasArrayReturn=false are deprecated (language ${languageId})`);
-};
-const safeTokenizeLine = (languageId, tokenizeLine, line, lineStateAtStart, hasArrayReturn) => {
-  try {
-    const lineState = tokenizeLine(line, lineStateAtStart);
-    if (!lineState?.tokens || !lineState.state) {
-      throw new Error('invalid tokenization result');
-    }
-    if (!hasArrayReturn) {
-      warnDeprecatedArrayReturn(languageId, tokenizeLine);
-      // workaround for old tokenizers
-      lineState.tokens = flattenTokensArray(lineState.tokens);
-    }
-    return lineState;
-  } catch (error) {
-    console.error(error);
-    return {
-      tokens: [/* type */0, /* length */line.length],
-      lineState: lineStateAtStart
-    };
-  }
-};
-
 const getLineInfos = (lines, tokenizer, languageId) => {
   const lineInfos = [];
   const {
@@ -8094,7 +8703,7 @@ const getLineInfos = (lines, tokenizer, languageId) => {
     const {
       tokens
     } = result;
-    const lineInfo = getLineInfo$1(line, tokens, TokenMap);
+    const lineInfo = getLineInfo(line, tokens, TokenMap);
     lineInfos.push(lineInfo);
     currentLineState = result;
   }
@@ -9379,7 +9988,7 @@ const createExtensionHostRpc = async () => {
 
 const initializeExtensionHost = async () => {
   const extensionHostRpc = await createExtensionHostRpc();
-  set$5(extensionHostRpc);
+  set$1(extensionHostRpc);
 };
 
 const sendMessagePortToSyntaxHighlightingWorker = async () => {
@@ -9399,586 +10008,92 @@ const sendMessagePortToSyntaxHighlightingWorker = async () => {
     } = getPortTuple();
     await invokeAndTransfer(
     // @ts-ignore
-    'SendMessagePortToSyntaxHighlightingWorker.sendMessagePortToSyntaxHighlightingWorker', port1, 'HandleMessagePort.handleMessagePort');
-    return port2;
-  }
-};
-
-const createSyntaxHighlightingWorkerRpc = async () => {
-  try {
-    const port = await sendMessagePortToSyntaxHighlightingWorker();
-    const rpc = await PlainMessagePortRpcParent.create({
-      commandMap: {},
-      messagePort: port
-    });
-    return rpc;
-  } catch (error) {
-    throw new VError(error, `Failed to create syntax highlighting worker rpc`);
-  }
-};
-
-let enabled = false;
-const setEnabled = value => {
-  enabled = value;
-};
-const getEnabled = () => {
-  return enabled;
-};
-
-const initializeSyntaxHighlighting = async (syntaxHighlightingEnabled, syncIncremental) => {
-  if (syntaxHighlightingEnabled) {
-    setEnabled$1(true);
-    const syntaxRpc = await createSyntaxHighlightingWorkerRpc();
-    set$3(syntaxRpc);
-  }
-  if (syncIncremental) {
-    setEnabled(true);
-  }
-};
-
-const intialize = async (syntaxHighlightingEnabled, syncIncremental) => {
-  await Promise.all([initializeSyntaxHighlighting(syntaxHighlightingEnabled, syncIncremental), initializeExtensionHost()]);
-};
-
-// TODO move cursor
-// TODO multiple cursors -> vscode removes multiple cursors
-// TODO with selection -> vscode moves whole selection
-const moveLineDown = editor => {
-  // const rowIndex = editor.cursor.rowIndex
-  // if (rowIndex === editor.lines.length - 1) {
-  //   return
-  // }
-  // const documentEdits = [
-  //   {
-  //     type: /* splice */ 2,
-  //     rowIndex: rowIndex,
-  //     count: 2,
-  //     newLines: [TextDocument.getLine(editor.textDocument, rowIndex + 1), TextDocument.getLine(editor.textDocument, rowIndex)],
-  //   },
-  // ]
-  // // @ts-ignore
-  // const cursorEdits = Editor.moveCursors(editor, (editor, cursor) => {
-  //   return {
-  //     rowIndex: cursor.rowIndex + 1,
-  //     columnIndex: cursor.columnIndex,
-  //   }
-  // })
-  // @ts-ignore
-  // Editor.scheduleDocumentAndCursors(editor, documentEdits, cursorEdits)
-  return editor;
-};
-
-// TODO handle multiple cursors
-const moveLineUp = editor => {
-  // const rowIndex = editor.cursor.rowIndex
-  // if (rowIndex === 0) {
-  //   return
-  // }
-  // const documentEdits = [
-  //   {
-  //     type: /* splice */ 2,
-  //     rowIndex: rowIndex - 1,
-  //     count: 2,
-  //     newLines: [TextDocument.getLine(editor.textDocument, rowIndex), TextDocument.getLine(editor.textDocument, rowIndex - 1)],
-  //   },
-  // ]
-  // // @ts-ignore
-  // const cursorEdits = Editor.moveCursors(editor, (editor, cursor) => {
-  //   return {
-  //     // TODO handle bottom 0
-  //     rowIndex: cursor.rowIndex - 1,
-  //     columnIndex: cursor.columnIndex,
-  //   }
-  // })
-  // // @ts-ignore
-  // Editor.scheduleDocumentAndCursors(editor, documentEdits, cursorEdits)
-  return editor;
-};
-
-const Link$1 = 'Link';
-const Function = 'Function';
-const Parameter = 'Parameter';
-const Type = 'Type';
-const VariableName = 'VariableName';
-const Class = 'Class';
-
-const Link = 1;
-const Ts2816 = 2816;
-const Ts2817 = 2817;
-const Ts2824 = 2824;
-const Ts2825 = 2825;
-const Ts2856 = 2956;
-const Ts2857 = 2857;
-const Ts3072 = 3072;
-const Ts3073 = 3073;
-const Ts3077 = 3077;
-const Ts3088 = 3088;
-const Ts1792 = 1792;
-const Ts1793 = 1793;
-const Ts512 = 512;
-const Ts513 = 513;
-const Ts769 = 769;
-const Ts1024 = 1024;
-const Ts1536 = 1536;
-const Ts1537 = 1537;
-const Ts1544 = 1544;
-const Ts1545 = 1545;
-const Ts2048 = 2048;
-const Ts2049 = 2049;
-const Ts2056 = 2056;
-const Ts2057 = 2057;
-const Ts2064 = 2064;
-const Ts2080 = 2080;
-const Ts2081 = 2081;
-const Ts2088 = 2088;
-const Ts2089 = 2089;
-const Ts2313 = 2313;
-const Ts2560 = 2560;
-const Ts2561 = 2561;
-const Ts2569 = 2569;
-const Ts2584 = 2584;
-const Ts256 = 256;
-const Ts257 = 257;
-const Ts272 = 272;
-
-const getDecorationClassName = type => {
-  switch (type) {
-    case Link:
-      return Link$1;
-    case Ts2816:
-    case Ts2817:
-    case Ts2824:
-    case Ts2825:
-    case Ts2856:
-    case Ts2857:
-    case Ts3072:
-    case Ts3073:
-    case Ts3077:
-    case Ts3088:
-      return Function;
-    case Ts1792:
-    case Ts1793:
-      return Parameter;
-    case Ts512:
-    case Ts513:
-    case Ts769:
-    case Ts1024:
-    case Ts1536:
-    case Ts1537:
-    case Ts1544:
-    case Ts1545:
-      return Type;
-    case Ts2048:
-    case Ts2049:
-    case Ts2056:
-    case Ts2057:
-    case Ts2064:
-    case Ts2080:
-    case Ts2081:
-    case Ts2088:
-    case Ts2089:
-    case Ts2313:
-    case Ts2560:
-    case Ts2561:
-    case Ts2569:
-    case Ts2584:
-      return VariableName;
-    case Ts256:
-    case Ts257:
-    case Ts272:
-      return Class;
-    default:
-      return `Unknown-${type}`;
-  }
-};
-
-const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
-  const tokenizersToLoad = [];
-  const embeddedResults = [];
-  let topContext;
-  for (const index of linesWithEmbed) {
-    const result = lineCache[index + 1];
-    const line = lines[index];
-    if (result.embeddedLanguage) {
-      const {
-        embeddedLanguage,
-        embeddedLanguageStart,
-        embeddedLanguageEnd
-      } = result;
-      const embeddedTokenizer = getTokenizer(embeddedLanguage);
-      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
-        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
-        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
-        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
-        topContext = embedResult;
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          result: embedResult,
-          TokenMap: embeddedTokenizer.TokenMap,
-          isFull
-        });
-      } else if (line.length === 0) {
-        const embedResult = {
-          tokens: []
-        };
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          result: embedResult,
-          isFull: true,
-          TokenMap: []
-        });
-      } else {
-        tokenizersToLoad.push(embeddedLanguage);
-        embeddedResults.push({
-          result: {},
-          isFull: false,
-          TokenMap: []
-        });
-        topContext = undefined;
-      }
-    } else {
-      topContext = undefined;
-    }
-  }
-  return {
-    tokenizersToLoad,
-    embeddedResults
-  };
-};
-const getTokenizeEndIndex = (invalidStartIndex, endLineIndex, tokenizeStartIndex) => {
-  return invalidStartIndex < endLineIndex ? endLineIndex : tokenizeStartIndex;
-};
-
-// TODO only send changed lines to renderer process instead of all lines in viewport
-const getTokensViewport = (editor, startLineIndex, endLineIndex) => {
-  const {
-    invalidStartIndex,
-    lineCache,
-    tokenizerId,
-    lines,
-    languageId
-  } = editor;
-  const tokenizer = get(tokenizerId);
-  const {
-    hasArrayReturn,
-    tokenizeLine,
-    initialLineState
-  } = tokenizer;
-  const tokenizeStartIndex = invalidStartIndex;
-  const tokenizeEndIndex = getTokenizeEndIndex(invalidStartIndex, endLineIndex, tokenizeStartIndex);
-  const tokenizersToLoad = [];
-  const embeddedResults = [];
-  const linesWithEmbed = [];
-  for (let i = tokenizeStartIndex; i < tokenizeEndIndex; i++) {
-    const lineState = i === 0 ? getInitialLineState(initialLineState) : lineCache[i];
-    const line = lines[i];
-    const result = safeTokenizeLine(languageId, tokenizeLine, line, lineState, hasArrayReturn);
-    // TODO if lineCacheEnd matches the one before, skip tokenizing lines after
-    lineCache[i + 1] = result;
-    if (result.embeddedLanguage) {
-      result.embeddedResultIndex = linesWithEmbed.length;
-      linesWithEmbed.push(i);
-    }
-  }
-  const visibleLines = lineCache.slice(startLineIndex + 1, endLineIndex + 1);
-  if (linesWithEmbed.length > 0) {
-    const {
-      tokenizersToLoad,
-      embeddedResults
-    } = getTokensViewportEmbedded(languageId, lines, lineCache, linesWithEmbed);
-    // TODO support lineCache with embedded content
-    editor.invalidStartIndex = 0;
-    return {
-      tokens: visibleLines,
-      tokenizersToLoad,
-      embeddedResults
-    };
-  }
-  editor.invalidStartIndex = Math.max(invalidStartIndex, tokenizeEndIndex);
-  return {
-    tokens: visibleLines,
-    tokenizersToLoad,
-    embeddedResults
-  };
-};
-
-const sentLines = Object.create(null);
-
-// TODO only send changed lines to renderer process instead of all lines in viewport
-const getTokensViewport2 = async (editor, startLineIndex, endLineIndex, syncIncremental) => {
-  if (getEnabled$1()) {
-    if (syncIncremental) {
-      const {
-        invalidStartIndex,
-        lines,
-        languageId,
-        id
-      } = editor;
-      let hasLinesToSend = true;
-      let linesToSend = lines;
-      if (sentLines[id] === lines) {
-        hasLinesToSend = false;
-        linesToSend = [];
-      } else {
-        sentLines[id] = lines;
-      }
-      const slimEditor = {
-        languageId,
-        invalidStartIndex
-      };
-      return invoke$3('GetTokensViewport.getTokensViewport', slimEditor,
-      // @ts-ignore
-      startLineIndex, endLineIndex, hasLinesToSend, id, linesToSend);
-    }
-    // TODO only send needed lines of text
-    // @ts-ignore
-    return invoke$3('GetTokensViewport.getTokensViewport', editor, startLineIndex, endLineIndex, true, editor.id, editor.lines);
-  }
-  return getTokensViewport(editor, startLineIndex, endLineIndex);
-};
-
-const loadTokenizers = async languageIds => {
-  for (const languageId of languageIds) {
-    // @ts-ignore
-    await loadTokenizer(languageId);
-  }
-};
-
-// const getTokensIncremental = (editor, min, max) => {
-//   const currentLength = editor.lineStateCache.length
-//   const tokens = []
-//   const lines = editor.lines
-//   let lineState = editor.tokenizer.initialLineState
-//   for (let i = currentLength; i < max; i++) {
-//     const line = lines[i]
-//     try {
-//       lineState = editor.tokenizer.tokenizeLine(line, lineState)
-//       if (!lineState || !lineState.tokens || !lineState.state) {
-//         throw new Error('invalid tokenization result')
-//       }
-//     } catch (error) {
-//       tokens.push([{ length: line.length, type: 0 }])
-//       console.error(error)
-//       // renderWithoutSyntaxHighlighting(state, firstRow, lastRow)
-//       continue
-//     }
-//     const newTokens = lineState.tokens
-//     tokens.push(newTokens)
-//   }
-//   return tokens
-// }
-
-// const getLineInfosIncremental = (editor, tokens, minLineY, maxLineY) => {
-//   const result = []
-//   const lines = editor.lines
-//   const TokenMap = editor.tokenizer.TokenMap
-//   for (let i = minLineY; i < maxLineY; i++) {
-//     result.push(getLineInfo(lines[i], tokens[i], TokenMap))
-//   }
-//   return result
-// }
-
-const getStartDefaults = (tokens, minOffset) => {
-  let start = 0;
-  let end = 0;
-  let startIndex = 0;
-  const tokensLength = tokens.length;
-  for (let i = 0; i < tokensLength; i += 2) {
-    const tokenLength = tokens[i + 1];
-    end += tokenLength;
-    start = end;
-    if (start >= minOffset) {
-      start -= tokenLength;
-      end -= tokenLength;
-      startIndex = i;
-      break;
-    }
-  }
-  return {
-    start,
-    end,
-    startIndex
-  };
-};
-const getLineInfoEmbeddedFull = (embeddedResults, tokenResults, line, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset) => {
-  const lineInfo = [];
-  const embeddedResult = embeddedResults[tokenResults.embeddedResultIndex];
-  const embeddedTokens = embeddedResult.result.tokens;
-  const embeddedTokenMap = embeddedResult.TokenMap;
-  const tokensLength = embeddedTokens.length;
-  let {
-    startIndex,
-    start,
-    end
-  } = getStartDefaults(embeddedTokens, minOffset);
-  const difference = getDifference(start, averageCharWidth, deltaX);
-  for (let i = startIndex; i < tokensLength; i += 2) {
-    const tokenType = embeddedTokens[i];
-    const tokenLength = embeddedTokens[i + 1];
-    end += tokenLength;
-    const className = `Token ${embeddedTokenMap[tokenType] || 'Unknown'}`;
-    const text = line.slice(start, end);
-    const normalizedText = normalizeText(text, normalize, tabSize);
-    lineInfo.push(normalizedText, className);
-    start = end;
-    if (end >= maxOffset) {
-      break;
-    }
+    'SendMessagePortToSyntaxHighlightingWorker.sendMessagePortToSyntaxHighlightingWorker', port1, 'HandleMessagePort.handleMessagePort');
+    return port2;
   }
-  return {
-    lineInfo,
-    difference
-  };
 };
-const getOffsets = (deltaX, width, averageCharWidth) => {
-  // TODO accurately measure char widths using offscreen canvas
-  // and use fast measurements for monospace ascii text
-  if (deltaX === 0) {
-    return {
-      minOffset: 0,
-      maxOffset: Math.ceil(width / averageCharWidth)
-    };
+
+const createSyntaxHighlightingWorkerRpc = async () => {
+  try {
+    const port = await sendMessagePortToSyntaxHighlightingWorker();
+    const rpc = await PlainMessagePortRpcParent.create({
+      commandMap: {},
+      messagePort: port
+    });
+    return rpc;
+  } catch (error) {
+    throw new VError(error, `Failed to create syntax highlighting worker rpc`);
   }
-  const minOffset = Math.ceil(deltaX / averageCharWidth);
-  const maxOffset = minOffset + Math.ceil(width / averageCharWidth);
-  return {
-    minOffset,
-    maxOffset
-  };
-};
-const getDifference = (start, averageCharWidth, deltaX) => {
-  const beforeWidth = start * averageCharWidth;
-  const difference = beforeWidth - deltaX;
-  return difference;
 };
-const getLineInfoDefault = (line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset) => {
-  const lineInfo = [];
-  let decorationIndex = 0;
-  for (; decorationIndex < decorations.length; decorationIndex += 3) {
-    const decorationOffset = decorations[decorationIndex];
-    if (decorationOffset >= lineOffset) {
-      break;
-    }
-  }
-  const {
-    tokens
-  } = tokenResults;
-  let {
-    startIndex,
-    start,
-    end
-  } = getStartDefaults(tokens, minOffset);
-  const difference = getDifference(start, averageCharWidth, deltaX);
-  const tokensLength = tokens.length;
-  for (let i = startIndex; i < tokensLength; i += 2) {
-    const tokenType = tokens[i];
-    const tokenLength = tokens[i + 1];
-    const decorationOffset = decorations[decorationIndex];
-    let extraClassName = '';
-    if (decorationOffset !== undefined && decorationOffset - lineOffset === start) {
-      // @ts-ignore
-      decorations[++decorationIndex];
-      const decorationType = decorations[++decorationIndex];
-      // @ts-ignore
-      decorations[++decorationIndex];
-      //   decorationIndex,
-      //   decorationLength,
-      //   decorationType,
-      //   decorationModifiers,
-      // })
-      extraClassName = getDecorationClassName(decorationType);
-    }
-    end += tokenLength;
-    const text = line.slice(start, end);
-    const className = `Token ${extraClassName || TokenMap[tokenType] || 'Unknown'}`;
-    const normalizedText = normalizeText(text, normalize, tabSize);
-    lineInfo.push(normalizedText, className);
-    start = end;
-    if (end >= maxOffset) {
-      break;
-    }
+
+const initializeSyntaxHighlighting = async (syntaxHighlightingEnabled, syncIncremental) => {
+  if (syntaxHighlightingEnabled) {
+    setEnabled$1(true);
+    const syntaxRpc = await createSyntaxHighlightingWorkerRpc();
+    set$5(syntaxRpc);
   }
-  return {
-    lineInfo,
-    difference
-  };
-};
-const getLineInfo = (line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth) => {
-  const {
-    minOffset,
-    maxOffset
-  } = getOffsets(deltaX, width, averageCharWidth);
-  if (embeddedResults.length > 0 && tokenResults.embeddedResultIndex !== undefined) {
-    const embeddedResult = embeddedResults[tokenResults.embeddedResultIndex];
-    if (embeddedResult?.isFull) {
-      return getLineInfoEmbeddedFull(embeddedResults, tokenResults, line, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset);
-    }
+  if (syncIncremental) {
+    setEnabled(true);
   }
-  return getLineInfoDefault(line, tokenResults, embeddedResults, decorations, TokenMap, lineOffset, normalize, tabSize, width, deltaX, averageCharWidth, minOffset, maxOffset);
 };
 
-// TODO need lots of tests for this
-const getLineInfosViewport = (editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth) => {
-  const result = [];
-  const differences = [];
-  const {
-    lines,
-    decorations,
-    languageId
-  } = editor;
-  const tokenMap = get$1(languageId);
-  let offset = minLineOffset;
-  const tabSize = 2;
-  for (let i = minLineY; i < maxLineY; i++) {
-    const line = lines[i];
-    const normalize = shouldNormalizeText(line);
-    const {
-      lineInfo,
-      difference
-    } = getLineInfo(line, tokens[i - minLineY], embeddedResults, decorations, tokenMap, offset, normalize, tabSize, width, deltaX, averageCharWidth);
-    result.push(lineInfo);
-    differences.push(difference);
-    offset += line.length + 1;
-  }
-  return {
-    result,
-    differences
-  };
+const intialize = async (syntaxHighlightingEnabled, syncIncremental) => {
+  await Promise.all([initializeSyntaxHighlighting(syntaxHighlightingEnabled, syncIncremental), initializeExtensionHost()]);
 };
-const getVisible = async (editor, syncIncremental) => {
-  // TODO should separate rendering from business logic somehow
-  // currently hard to test because need to mock editor height, top, left,
-  // invalidStartIndex, lineCache, etc. just for testing editorType
-  // editor.invalidStartIndex = changes[0].start.rowIndex
-  // @ts-ignore
-  const {
-    minLineY,
-    numberOfVisibleLines,
-    lines,
-    width,
-    deltaX,
-    charWidth
-  } = editor;
-  const maxLineY = Math.min(minLineY + numberOfVisibleLines, lines.length);
+
+// TODO move cursor
+// TODO multiple cursors -> vscode removes multiple cursors
+// TODO with selection -> vscode moves whole selection
+const moveLineDown = editor => {
+  // const rowIndex = editor.cursor.rowIndex
+  // if (rowIndex === editor.lines.length - 1) {
+  //   return
+  // }
+  // const documentEdits = [
+  //   {
+  //     type: /* splice */ 2,
+  //     rowIndex: rowIndex,
+  //     count: 2,
+  //     newLines: [TextDocument.getLine(editor.textDocument, rowIndex + 1), TextDocument.getLine(editor.textDocument, rowIndex)],
+  //   },
+  // ]
+  // // @ts-ignore
+  // const cursorEdits = Editor.moveCursors(editor, (editor, cursor) => {
+  //   return {
+  //     rowIndex: cursor.rowIndex + 1,
+  //     columnIndex: cursor.columnIndex,
+  //   }
+  // })
   // @ts-ignore
-  const {
-    tokens,
-    tokenizersToLoad,
-    embeddedResults
-  } = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
-  const minLineOffset = offsetAtSync(editor, minLineY, 0);
-  const averageCharWidth = charWidth;
-  const {
-    result,
-    differences
-  } = getLineInfosViewport(editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth);
-  if (tokenizersToLoad.length > 0) {
-    loadTokenizers(tokenizersToLoad);
-  }
-  return {
-    textInfos: result,
-    differences
-  };
+  // Editor.scheduleDocumentAndCursors(editor, documentEdits, cursorEdits)
+  return editor;
+};
+
+// TODO handle multiple cursors
+const moveLineUp = editor => {
+  // const rowIndex = editor.cursor.rowIndex
+  // if (rowIndex === 0) {
+  //   return
+  // }
+  // const documentEdits = [
+  //   {
+  //     type: /* splice */ 2,
+  //     rowIndex: rowIndex - 1,
+  //     count: 2,
+  //     newLines: [TextDocument.getLine(editor.textDocument, rowIndex), TextDocument.getLine(editor.textDocument, rowIndex - 1)],
+  //   },
+  // ]
+  // // @ts-ignore
+  // const cursorEdits = Editor.moveCursors(editor, (editor, cursor) => {
+  //   return {
+  //     // TODO handle bottom 0
+  //     rowIndex: cursor.rowIndex - 1,
+  //     columnIndex: cursor.columnIndex,
+  //   }
+  // })
+  // // @ts-ignore
+  // Editor.scheduleDocumentAndCursors(editor, documentEdits, cursorEdits)
+  return editor;
 };
 
 const getCursorsVirtualDom = cursors => {
@@ -10077,36 +10192,6 @@ const getEditorRowsVirtualDom = (textInfos, differences, lineNumbers = true, hig
   return dom;
 };
 
-const getIncrementalEdits = async (oldState, newState) => {
-  if (!newState.undoStack) {
-    return undefined;
-  }
-  const lastChanges = newState.undoStack.at(-1);
-  if (lastChanges && lastChanges.length === 1) {
-    const lastChange = lastChanges[0];
-    if (lastChange.origin === EditorType) {
-      const {
-        rowIndex
-      } = lastChange.start;
-      const {
-        lines
-      } = newState;
-      const oldLine = oldState.lines[rowIndex];
-      const newLine = lines[rowIndex];
-      // @ts-ignore
-      const incrementalEdits = await invoke$3(
-      // @ts-ignore
-      'TokenizeIncremental.tokenizeIncremental', newState.uid,
-      // @ts-ignore
-      newState.languageId, oldLine, newLine, rowIndex, newState.minLineY);
-      if (incrementalEdits && incrementalEdits.length === 1) {
-        return incrementalEdits;
-      }
-    }
-  }
-  return undefined;
-};
-
 const getSelectionsVirtualDom = selections => {
   const dom = [];
   for (let i = 0; i < selections.length; i += 4) {
@@ -10152,16 +10237,17 @@ const renderLines = {
   isEqual(oldState, newState) {
     return oldState.lines === newState.lines && oldState.tokenizerId === newState.tokenizerId && oldState.minLineY === newState.minLineY && oldState.decorations === newState.decorations && oldState.embeds === newState.embeds && oldState.deltaX === newState.deltaX && oldState.width === newState.width && oldState.highlightedLine === newState.highlightedLine && oldState.debugEnabled === newState.debugEnabled;
   },
-  async apply(oldState, newState) {
-    const incrementalEdits = await getIncrementalEdits(oldState, newState);
-    if (incrementalEdits) {
+  apply(oldState, newState) {
+    const {
+      incrementalEdits
+    } = newState;
+    if (incrementalEdits !== emptyIncrementalEdits) {
       return [/* method */'setIncrementalEdits', /* incrementalEdits */incrementalEdits];
     }
-    const syncIncremental = getEnabled();
     const {
       textInfos,
       differences
-    } = await getVisible(newState, syncIncremental);
+    } = newState;
     newState.differences = differences;
     const {
       highlightedLine,
@@ -10180,7 +10266,7 @@ const renderSelections = {
     const {
       cursorInfos,
       selectionInfos
-    } = getVisible$1(newState);
+    } = getVisible(newState);
     const cursorsDom = getCursorsVirtualDom(cursorInfos);
     const selectionsDom = getSelectionsVirtualDom(selectionInfos);
     return [/* method */'setSelections', cursorsDom, selectionsDom];
@@ -10306,7 +10392,7 @@ const renderWidgets = {
   multiple: true
 };
 const render$6 = [renderLines, renderSelections, renderScrollBarX, renderScrollBarY, renderFocus$1, renderDecorations, renderGutterInfo, renderWidgets];
-const renderEditor = async id => {
+const renderEditor = id => {
   const instance = get$4(id);
   if (!instance) {
     return [];
@@ -10319,7 +10405,7 @@ const renderEditor = async id => {
   set$6(id, newState, newState);
   for (const item of render$6) {
     if (!item.isEqual(oldState, newState)) {
-      const result = await item.apply(oldState, newState);
+      const result = item.apply(oldState, newState);
       // @ts-ignore
       if (item.multiple) {
         commands.push(...result);
@@ -10456,11 +10542,16 @@ const wrapCommand = fn => async (editorUid, ...args) => {
       effect.apply(newEditor);
     }
   }
+  // TODO if editor did not change, no need to update furthur
+
+  // TODO combine neweditor with latest editor?
+
   set$6(editorUid, oldInstance.newState, newEditor);
-  // TODO if possible, rendering should be sync
-  const commands = await renderEditor(editorUid);
-  newEditor.commands = commands;
-  return newEditor;
+  const commands = renderEditor(editorUid);
+  return {
+    ...newEditor,
+    commands
+  };
 };
 const wrapCommands = commands => {
   for (const [key, value] of Object.entries(commands)) {
@@ -10695,6 +10786,7 @@ const commandMap = {
   'FindWidget.toggleUseRegularExpression': toggleUseRegularExpression,
   'FindWidget.focusNextElement': focusNextElement,
   'FindWidget.focusPreviousElement': focusPreviousElement,
+  'FindWidget.togglePreserveCase': togglePreserveCase,
   'Font.ensure': ensure,
   'HandleMessagePort.handleMessagePort': handleMessagePort,
   'Hover.getHoverInfo': getEditorHoverInfo,