@lvce-editor/static-server 0.44.5 → 0.44.7

package/static/{10163fd → 824bfc9}/packages/syntax-highlighting-worker/dist/syntaxHighlightingWorkerMain.js

@@ -1,32 +1,58 @@
-const commands = Object.create(null);
-const register = commandMap => {
-  Object.assign(commands, commandMap);
-};
-const getCommand = key => {
-  return commands[key];
+const normalizeLine = line => {
+  if (line.startsWith('Error: ')) {
+    return line.slice('Error: '.length);
+  }
+  if (line.startsWith('VError: ')) {
+    return line.slice('VError: '.length);
+  }
+  return line;
 };
-const execute = (command, ...args) => {
-  const fn = getCommand(command);
-  if (!fn) {
-    throw new Error(`command not found ${command}`);
+const getCombinedMessage = (error, message) => {
+  const stringifiedError = normalizeLine(`${error}`);
+  if (message) {
+    return `${message}: ${stringifiedError}`;
   }
-  return fn(...args);
+  return stringifiedError;
 };
-
-const GetTokensViewport = 'GetTokensViewport.getTokensViewport';
-const HandleMessagePort = 'HandleMessagePort.handleMessagePort';
-const LoadTokenizer = 'Tokenizer.load';
-const SetLines = 'TextDocument.setLines';
-const TokenizeCodeBlock = 'Tokenizer.tokenizeCodeBlock';
-const TokenizeIncremental = 'TokenizeIncremental.tokenizeIncremental';
-
-const deepCopy = value => {
-  return structuredClone(value);
+const NewLine$3 = '\n';
+const getNewLineIndex$1 = (string, startIndex = undefined) => {
+  return string.indexOf(NewLine$3, startIndex);
 };
-
-const getInitialLineState = initialLineState => {
-  return deepCopy(initialLineState);
+const mergeStacks = (parent, child) => {
+  if (!child) {
+    return parent;
+  }
+  const parentNewLineIndex = getNewLineIndex$1(parent);
+  const childNewLineIndex = getNewLineIndex$1(child);
+  if (childNewLineIndex === -1) {
+    return parent;
+  }
+  const parentFirstLine = parent.slice(0, parentNewLineIndex);
+  const childRest = child.slice(childNewLineIndex);
+  const childFirstLine = normalizeLine(child.slice(0, childNewLineIndex));
+  if (parentFirstLine.includes(childFirstLine)) {
+    return parentFirstLine + childRest;
+  }
+  return child;
 };
+class VError extends Error {
+  constructor(error, message) {
+    const combinedMessage = getCombinedMessage(error, message);
+    super(combinedMessage);
+    this.name = 'VError';
+    if (error instanceof Error) {
+      this.stack = mergeStacks(this.stack, error.stack);
+    }
+    if (error.codeFrame) {
+      // @ts-ignore
+      this.codeFrame = error.codeFrame;
+    }
+    if (error.code) {
+      // @ts-ignore
+      this.code = error.code;
+    }
+  }
+}
 
 class AssertionError extends Error {
   constructor(message) {
@@ -63,298 +89,557 @@ const object = value => {
   }
 };
 
-const state = {
-  warned: []
+const isMessagePort = value => {
+  return value && value instanceof MessagePort;
 };
-const flattenTokensArray = tokens => {
-  const flattened = [];
-  for (const token of tokens) {
-    object(token);
-    flattened.push(token.type, token.length);
+const isMessagePortMain = value => {
+  return value && value.constructor && value.constructor.name === 'MessagePortMain';
+};
+const isOffscreenCanvas = value => {
+  return typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas;
+};
+const isInstanceOf = (value, constructorName) => {
+  return value?.constructor?.name === constructorName;
+};
+const isSocket = value => {
+  return isInstanceOf(value, 'Socket');
+};
+const transferrables = [isMessagePort, isMessagePortMain, isOffscreenCanvas, isSocket];
+const isTransferrable = value => {
+  for (const fn of transferrables) {
+    if (fn(value)) {
+      return true;
+    }
   }
-  return flattened;
+  return false;
 };
-const warnDeprecatedArrayReturn = (languageId, fn) => {
-  if (state.warned.includes(fn)) {
+const walkValue = (value, transferrables, isTransferrable) => {
+  if (!value) {
     return;
   }
-  state.warned.push(fn);
-  console.warn(`tokenizers without hasArrayReturn=false are deprecated (language ${languageId})`);
-};
-/**
- *
- * @param {(line:string, lineState)=>any} tokenizeLine
- * @param {string} line
- * @param {any} lineStateAtStart
- * @param {boolean} hasArrayReturn
- * @returns
- */
-const safeTokenizeLine = (languageId, tokenizeLine, line, lineStateAtStart, hasArrayReturn) => {
-  try {
-    const lineState = tokenizeLine(line, lineStateAtStart);
-    if (!lineState || !lineState.tokens || !lineState.state) {
-      throw new Error('invalid tokenization result');
+  if (isTransferrable(value)) {
+    transferrables.push(value);
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      walkValue(item, transferrables, isTransferrable);
     }
-    if (!hasArrayReturn) {
-      warnDeprecatedArrayReturn(languageId, tokenizeLine);
-      // workaround for old tokenizers
-      lineState.tokens = flattenTokensArray(lineState.tokens);
+    return;
+  }
+  if (typeof value === 'object') {
+    for (const property of Object.values(value)) {
+      walkValue(property, transferrables, isTransferrable);
     }
-    return lineState;
-  } catch (error) {
-    console.error(error);
-    return {
-      tokens: [/* type */0, /* length */line.length],
-      lineState: lineStateAtStart
-    };
+    return;
   }
 };
-
-const lineMap = Object.create(null);
-const lineCaches = Object.create(null);
-const invalidStartIndices = Object.create(null);
-const languageIds = Object.create(null);
-const setLines = (editorId, lines) => {
-  lineMap[editorId] = lines;
-  lineCaches[editorId] = [];
-  invalidStartIndices[editorId] = 0;
+const getTransferrables = value => {
+  const transferrables = [];
+  walkValue(value, transferrables, isTransferrable);
+  return transferrables;
 };
-const getLines = editorId => {
-  return lineMap[editorId];
+const attachEvents = that => {
+  const handleMessage = (...args) => {
+    const data = that.getData(...args);
+    that.dispatchEvent(new MessageEvent('message', {
+      data
+    }));
+  };
+  that.onMessage(handleMessage);
+  const handleClose = event => {
+    that.dispatchEvent(new Event('close'));
+  };
+  that.onClose(handleClose);
 };
-const getLineCache = editorId => {
-  return lineCaches[editorId];
+class Ipc extends EventTarget {
+  constructor(rawIpc) {
+    super();
+    this._rawIpc = rawIpc;
+    attachEvents(this);
+  }
+}
+const E_INCOMPATIBLE_NATIVE_MODULE = 'E_INCOMPATIBLE_NATIVE_MODULE';
+const E_MODULES_NOT_SUPPORTED_IN_ELECTRON = 'E_MODULES_NOT_SUPPORTED_IN_ELECTRON';
+const ERR_MODULE_NOT_FOUND = 'ERR_MODULE_NOT_FOUND';
+const NewLine$2 = '\n';
+const joinLines$1 = lines => {
+  return lines.join(NewLine$2);
 };
-const getInvalidStartIndex = editorId => {
-  return invalidStartIndices[editorId];
+const RE_AT = /^\s+at/;
+const RE_AT_PROMISE_INDEX = /^\s*at async Promise.all \(index \d+\)$/;
+const isNormalStackLine = line => {
+  return RE_AT.test(line) && !RE_AT_PROMISE_INDEX.test(line);
 };
-const resetInvalidStartIndex = languageId => {
-  for (const [key, value] of Object.entries(languageIds)) {
-    if (value === languageId) {
-      invalidStartIndices[key] = 0;
+const getDetails = lines => {
+  const index = lines.findIndex(isNormalStackLine);
+  if (index === -1) {
+    return {
+      actualMessage: joinLines$1(lines),
+      rest: []
+    };
+  }
+  let lastIndex = index - 1;
+  while (++lastIndex < lines.length) {
+    if (!isNormalStackLine(lines[lastIndex])) {
+      break;
     }
   }
+  return {
+    actualMessage: lines[index - 1],
+    rest: lines.slice(index, lastIndex)
+  };
 };
-const setInvalidStartIndex = (editorId, index) => {
-  invalidStartIndices[editorId] = index;
-};
-const setLanguageId = (editorId, languageId) => {
-  languageIds[editorId] = languageId;
+const splitLines$2 = lines => {
+  return lines.split(NewLine$2);
 };
-
-/**
- * @enum number
- */
-const State = {
-  TopLevelContent: 1
+const RE_MESSAGE_CODE_BLOCK_START = /^Error: The module '.*'$/;
+const RE_MESSAGE_CODE_BLOCK_END = /^\s* at/;
+const isMessageCodeBlockStartIndex = line => {
+  return RE_MESSAGE_CODE_BLOCK_START.test(line);
 };
-
-/**
- * @enum number
- */
-const TokenType = {
-  Text: 1
+const isMessageCodeBlockEndIndex = line => {
+  return RE_MESSAGE_CODE_BLOCK_END.test(line);
 };
-const TokenMap = {
-  [TokenType.Text]: 'Text'
+const getMessageCodeBlock = stderr => {
+  const lines = splitLines$2(stderr);
+  const startIndex = lines.findIndex(isMessageCodeBlockStartIndex);
+  const endIndex = startIndex + lines.slice(startIndex).findIndex(isMessageCodeBlockEndIndex, startIndex);
+  const relevantLines = lines.slice(startIndex, endIndex);
+  const relevantMessage = relevantLines.join(' ').slice('Error: '.length);
+  return relevantMessage;
 };
-const initialLineState = {
-  state: State.TopLevelContent
+const isModuleNotFoundMessage = line => {
+  return line.includes('[ERR_MODULE_NOT_FOUND]');
 };
-const hasArrayReturn = true;
-const tokenizeLine = (line, lineState) => {
+const getModuleNotFoundError = stderr => {
+  const lines = splitLines$2(stderr);
+  const messageIndex = lines.findIndex(isModuleNotFoundMessage);
+  const message = lines[messageIndex];
   return {
-    tokens: [TokenType.Text, line.length],
-    state: lineState.state
+    message,
+    code: ERR_MODULE_NOT_FOUND
   };
 };
-
-const TokenizePlainText = {
-  __proto__: null,
-  State,
-  TokenMap,
-  TokenType,
-  hasArrayReturn,
-  initialLineState,
-  tokenizeLine
+const isModuleNotFoundError = stderr => {
+  if (!stderr) {
+    return false;
+  }
+  return stderr.includes('ERR_MODULE_NOT_FOUND');
 };
-
-const tokenizers = Object.create(null);
-const set = (id, value) => {
-  tokenizers[id] = value;
-  resetInvalidStartIndex(id);
+const isModulesSyntaxError = stderr => {
+  if (!stderr) {
+    return false;
+  }
+  return stderr.includes('SyntaxError: Cannot use import statement outside a module');
 };
-const get$1 = id => {
-  return tokenizers[id] || TokenizePlainText;
+const RE_NATIVE_MODULE_ERROR = /^innerError Error: Cannot find module '.*.node'/;
+const RE_NATIVE_MODULE_ERROR_2 = /was compiled against a different Node.js version/;
+const isUnhelpfulNativeModuleError = stderr => {
+  return RE_NATIVE_MODULE_ERROR.test(stderr) && RE_NATIVE_MODULE_ERROR_2.test(stderr);
 };
-
-// TODO loadTokenizer should be invoked from renderer worker
-const loadTokenizer = async (languageId, tokenizePath) => {
-  if (!tokenizePath) {
-    return;
+const getNativeModuleErrorMessage = stderr => {
+  const message = getMessageCodeBlock(stderr);
+  return {
+    message: `Incompatible native node module: ${message}`,
+    code: E_INCOMPATIBLE_NATIVE_MODULE
+  };
+};
+const getModuleSyntaxError = () => {
+  return {
+    message: `ES Modules are not supported in electron`,
+    code: E_MODULES_NOT_SUPPORTED_IN_ELECTRON
+  };
+};
+const getHelpfulChildProcessError = (stdout, stderr) => {
+  if (isUnhelpfulNativeModuleError(stderr)) {
+    return getNativeModuleErrorMessage(stderr);
   }
-  try {
-    // TODO check that tokenizer is valid
-    // 1. tokenizeLine should be of type function
-    // 2. getTokenClass should be of type function
-    const tokenizer = await import(tokenizePath);
-    if (typeof tokenizer.tokenizeLine !== 'function') {
-      console.warn(`tokenizer.tokenizeLine should be a function in "${tokenizePath}"`);
-      return;
-    }
-    if (!tokenizer.TokenMap || typeof tokenizer.TokenMap !== 'object' || Array.isArray(tokenizer.TokenMap)) {
-      console.warn(`tokenizer.TokenMap should be an object in "${tokenizePath}"`);
-      return;
-    }
-    set(languageId, tokenizer);
-    return tokenizer.TokenMap;
-  } catch (error) {
-    // TODO better error handling
-    console.error(error);
-    return;
+  if (isModulesSyntaxError(stderr)) {
+    return getModuleSyntaxError();
   }
-};
-const getTokenizer = languageId => {
-  if (get$1(languageId)) {
-    return get$1(languageId);
+  if (isModuleNotFoundError(stderr)) {
+    return getModuleNotFoundError(stderr);
   }
-  return TokenizePlainText;
+  const lines = splitLines$2(stderr);
+  const {
+    actualMessage,
+    rest
+  } = getDetails(lines);
+  return {
+    message: actualMessage,
+    code: '',
+    stack: rest
+  };
 };
-
-const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
-  const tokenizersToLoad = [];
-  const embeddedResults = [];
-  let topContext;
-  for (const index of linesWithEmbed) {
-    const result = lineCache[index + 1];
-    const line = lines[index];
-    if (result.embeddedLanguage) {
+class IpcError extends VError {
+  // @ts-ignore
+  constructor(betterMessage, stdout = '', stderr = '') {
+    if (stdout || stderr) {
+      // @ts-ignore
       const {
-        embeddedLanguage,
-        embeddedLanguageStart,
-        embeddedLanguageEnd
-      } = result;
-      const embeddedTokenizer = getTokenizer(embeddedLanguage);
-      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
-        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
-        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
-        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
-        topContext = embedResult;
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          result: embedResult,
-          TokenMap: embeddedTokenizer.TokenMap,
-          isFull
-        });
-      } else if (line.length === 0) {
-        const embedResult = {
-          tokens: []
-        };
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          result: embedResult,
-          isFull: true,
-          TokenMap: []
-        });
-      } else {
-        tokenizersToLoad.push(embeddedLanguage);
-        embeddedResults.push({
-          result: {},
-          isFull: false,
-          TokenMap: []
-        });
-        topContext = undefined;
-      }
+        message,
+        code,
+        stack
+      } = getHelpfulChildProcessError(stdout, stderr);
+      const cause = new Error(message);
+      // @ts-ignore
+      cause.code = code;
+      cause.stack = stack;
+      super(cause, betterMessage);
     } else {
-      topContext = undefined;
+      super(betterMessage);
     }
+    // @ts-ignore
+    this.name = 'IpcError';
+    // @ts-ignore
+    this.stdout = stdout;
+    // @ts-ignore
+    this.stderr = stderr;
   }
+}
+const readyMessage = 'ready';
+const getData$2 = event => {
+  return event.data;
+};
+const listen$8 = ({
+  port
+}) => {
+  return port;
+};
+const signal$9 = port => {
+  port.postMessage(readyMessage);
+};
+class IpcChildWithMessagePort extends Ipc {
+  getData(event) {
+    return getData$2(event);
+  }
+  send(message) {
+    this._rawIpc.postMessage(message);
+  }
+  sendAndTransfer(message) {
+    const transfer = getTransferrables(message);
+    this._rawIpc.postMessage(message, transfer);
+  }
+  dispose() {
+    // ignore
+  }
+  onClose(callback) {
+    // ignore
+  }
+  onMessage(callback) {
+    this._rawIpc.addEventListener('message', callback);
+    this._rawIpc.start();
+  }
+}
+const wrap$g = port => {
+  return new IpcChildWithMessagePort(port);
+};
+const IpcChildWithMessagePort$1 = {
+  __proto__: null,
+  listen: listen$8,
+  signal: signal$9,
+  wrap: wrap$g
+};
+const listen$7 = () => {
+  // @ts-ignore
+  if (typeof WorkerGlobalScope === 'undefined') {
+    throw new TypeError('module is not in web worker scope');
+  }
+  return globalThis;
+};
+const signal$8 = global => {
+  global.postMessage(readyMessage);
+};
+class IpcChildWithModuleWorker extends Ipc {
+  getData(event) {
+    return getData$2(event);
+  }
+  send(message) {
+    // @ts-ignore
+    this._rawIpc.postMessage(message);
+  }
+  sendAndTransfer(message) {
+    const transfer = getTransferrables(message);
+    // @ts-ignore
+    this._rawIpc.postMessage(message, transfer);
+  }
+  dispose() {
+    // ignore
+  }
+  onClose(callback) {
+    // ignore
+  }
+  onMessage(callback) {
+    this._rawIpc.addEventListener('message', callback);
+  }
+}
+const wrap$f = global => {
+  return new IpcChildWithModuleWorker(global);
+};
+const withResolvers = () => {
+  let _resolve;
+  const promise = new Promise(resolve => {
+    _resolve = resolve;
+  });
   return {
-    tokenizersToLoad,
-    embeddedResults
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    resolve: _resolve,
+    promise
   };
 };
-const getTokenizeEndIndex = (invalidStartIndex, endLineIndex, tokenizeStartIndex) => {
-  return invalidStartIndex < endLineIndex ? endLineIndex : tokenizeStartIndex;
+const waitForFirstMessage = async port => {
+  const {
+    resolve,
+    promise
+  } = withResolvers();
+  port.addEventListener('message', resolve, {
+    once: true
+  });
+  const event = await promise;
+  // @ts-ignore
+  return event.data;
 };
-
-// TODO only send changed lines to renderer process instead of all lines in viewport
-const getTokensViewport = (editor, startLineIndex, endLineIndex, hasLinesToSend, id, linesToSend) => {
-  if (hasLinesToSend) {
-    setLines(id, linesToSend);
+const listen$6 = async () => {
+  const parentIpcRaw = listen$7();
+  signal$8(parentIpcRaw);
+  const parentIpc = wrap$f(parentIpcRaw);
+  const firstMessage = await waitForFirstMessage(parentIpc);
+  if (firstMessage.method !== 'initialize') {
+    throw new IpcError('unexpected first message');
   }
-  const {
-    languageId
-  } = editor;
-  setLanguageId(id, languageId);
-  const lines = getLines(id);
-  const lineCache = getLineCache(id);
-  const invalidStartIndex = getInvalidStartIndex(id);
-  const tokenizer = get$1(languageId);
-  const {
-    hasArrayReturn,
-    tokenizeLine,
-    initialLineState
-  } = tokenizer;
-  const tokenizeStartIndex = invalidStartIndex;
-  const tokenizeEndIndex = getTokenizeEndIndex(invalidStartIndex, endLineIndex, tokenizeStartIndex);
-  const tokenizersToLoad = [];
-  const embeddedResults = [];
-  const linesWithEmbed = [];
-  for (let i = tokenizeStartIndex; i < tokenizeEndIndex; i++) {
-    const lineState = i === 0 ? getInitialLineState(initialLineState) : lineCache[i];
-    const line = lines[i];
-    const result = safeTokenizeLine(languageId, tokenizeLine, line, lineState, hasArrayReturn);
-    // TODO if lineCacheEnd matches the one before, skip tokenizing lines after
-    lineCache[i + 1] = result;
-    if (result.embeddedLanguage) {
-      result.embeddedResultIndex = linesWithEmbed.length;
-      linesWithEmbed.push(i);
+  const type = firstMessage.params[0];
+  if (type === 'message-port') {
+    parentIpc.send({
+      jsonrpc: '2.0',
+      id: firstMessage.id,
+      result: null
+    });
+    parentIpc.dispose();
+    const port = firstMessage.params[1];
+    return port;
+  }
+  return globalThis;
+};
+class IpcChildWithModuleWorkerAndMessagePort extends Ipc {
+  getData(event) {
+    return getData$2(event);
+  }
+  send(message) {
+    this._rawIpc.postMessage(message);
+  }
+  sendAndTransfer(message) {
+    const transfer = getTransferrables(message);
+    this._rawIpc.postMessage(message, transfer);
+  }
+  dispose() {
+    if (this._rawIpc.close) {
+      this._rawIpc.close();
     }
   }
-  const visibleLines = lineCache.slice(startLineIndex + 1, endLineIndex + 1);
-  if (linesWithEmbed.length > 0) {
-    const {
-      tokenizersToLoad,
-      embeddedResults
-    } = getTokensViewportEmbedded(languageId, lines, lineCache, linesWithEmbed);
-    // TODO support lineCache with embedded content
-    setInvalidStartIndex(id, 0);
-    return {
-      tokens: visibleLines,
-      tokenizersToLoad,
-      embeddedResults
-    };
+  onClose(callback) {
+    // ignore
+  }
+  onMessage(callback) {
+    this._rawIpc.addEventListener('message', callback);
+    this._rawIpc.start();
   }
-  setInvalidStartIndex(id, Math.max(invalidStartIndex, tokenizeEndIndex));
-  return {
-    tokens: visibleLines,
-    tokenizersToLoad,
-    embeddedResults
-  };
+}
+const wrap$e = port => {
+  return new IpcChildWithModuleWorkerAndMessagePort(port);
+};
+const IpcChildWithModuleWorkerAndMessagePort$1 = {
+  __proto__: null,
+  listen: listen$6,
+  wrap: wrap$e
 };
 
 const Two = '2.0';
+const create$4 = (method, params) => {
+  return {
+    jsonrpc: Two,
+    method,
+    params
+  };
+};
 const callbacks = Object.create(null);
-const get = id => {
+const set$2 = (id, fn) => {
+  callbacks[id] = fn;
+};
+const get$1 = id => {
   return callbacks[id];
 };
 const remove = id => {
   delete callbacks[id];
 };
+let id = 0;
+const create$3 = () => {
+  return ++id;
+};
+const registerPromise = () => {
+  const id = create$3();
+  const {
+    resolve,
+    promise
+  } = Promise.withResolvers();
+  set$2(id, resolve);
+  return {
+    id,
+    promise
+  };
+};
+const create$2 = (method, params) => {
+  const {
+    id,
+    promise
+  } = registerPromise();
+  const message = {
+    jsonrpc: Two,
+    method,
+    params,
+    id
+  };
+  return {
+    message,
+    promise
+  };
+};
 class JsonRpcError extends Error {
   constructor(message) {
     super(message);
     this.name = 'JsonRpcError';
   }
 }
+const NewLine$1 = '\n';
+const DomException = 'DOMException';
+const ReferenceError$1 = 'ReferenceError';
+const SyntaxError$1 = 'SyntaxError';
+const TypeError$1 = 'TypeError';
+const getErrorConstructor = (message, type) => {
+  if (type) {
+    switch (type) {
+      case DomException:
+        return DOMException;
+      case TypeError$1:
+        return TypeError;
+      case SyntaxError$1:
+        return SyntaxError;
+      case ReferenceError$1:
+        return ReferenceError;
+      default:
+        return Error;
+    }
+  }
+  if (message.startsWith('TypeError: ')) {
+    return TypeError;
+  }
+  if (message.startsWith('SyntaxError: ')) {
+    return SyntaxError;
+  }
+  if (message.startsWith('ReferenceError: ')) {
+    return ReferenceError;
+  }
+  return Error;
+};
+const constructError = (message, type, name) => {
+  const ErrorConstructor = getErrorConstructor(message, type);
+  if (ErrorConstructor === DOMException && name) {
+    return new ErrorConstructor(message, name);
+  }
+  if (ErrorConstructor === Error) {
+    const error = new Error(message);
+    if (name && name !== 'VError') {
+      error.name = name;
+    }
+    return error;
+  }
+  return new ErrorConstructor(message);
+};
+const getNewLineIndex = (string, startIndex = undefined) => {
+  return string.indexOf(NewLine$1, startIndex);
+};
+const getParentStack = error => {
+  let parentStack = error.stack || error.data || error.message || '';
+  if (parentStack.startsWith('    at')) {
+    parentStack = error.message + NewLine$1 + parentStack;
+  }
+  return parentStack;
+};
+const joinLines = lines => {
+  return lines.join(NewLine$1);
+};
 const MethodNotFound = -32601;
 const Custom = -32001;
+const splitLines$1 = lines => {
+  return lines.split(NewLine$1);
+};
+const restoreJsonRpcError = error => {
+  if (error && error instanceof Error) {
+    return error;
+  }
+  const currentStack = joinLines(splitLines$1(new Error().stack || '').slice(1));
+  if (error && error.code && error.code === MethodNotFound) {
+    const restoredError = new JsonRpcError(error.message);
+    const parentStack = getParentStack(error);
+    restoredError.stack = parentStack + NewLine$1 + currentStack;
+    return restoredError;
+  }
+  if (error && error.message) {
+    const restoredError = constructError(error.message, error.type, error.name);
+    if (error.data) {
+      if (error.data.stack && error.data.type && error.message) {
+        restoredError.stack = error.data.type + ': ' + error.message + NewLine$1 + error.data.stack + NewLine$1 + currentStack;
+      } else if (error.data.stack) {
+        restoredError.stack = error.data.stack;
+      }
+      if (error.data.codeFrame) {
+        // @ts-ignore
+        restoredError.codeFrame = error.data.codeFrame;
+      }
+      if (error.data.code) {
+        // @ts-ignore
+        restoredError.code = error.data.code;
+      }
+      if (error.data.type) {
+        // @ts-ignore
+        restoredError.name = error.data.type;
+      }
+    } else {
+      if (error.stack) {
+        const lowerStack = restoredError.stack || '';
+        // @ts-ignore
+        const indexNewLine = getNewLineIndex(lowerStack);
+        const parentStack = getParentStack(error);
+        // @ts-ignore
+        restoredError.stack = parentStack + lowerStack.slice(indexNewLine);
+      }
+      if (error.codeFrame) {
+        // @ts-ignore
+        restoredError.codeFrame = error.codeFrame;
+      }
+    }
+    return restoredError;
+  }
+  if (typeof error === 'string') {
+    return new Error(`JsonRpc Error: ${error}`);
+  }
+  return new Error(`JsonRpc Error: ${error}`);
+};
+const unwrapJsonRpcResult = responseMessage => {
+  if ('error' in responseMessage) {
+    const restoredError = restoreJsonRpcError(responseMessage.error);
+    throw restoredError;
+  }
+  if ('result' in responseMessage) {
+    return responseMessage.result;
+  }
+  throw new JsonRpcError('unexpected response message');
+};
 const warn = (...args) => {
   console.warn(...args);
 };
 const resolve = (id, response) => {
-  const fn = get(id);
+  const fn = get$1(id);
   if (!fn) {
     console.log(response);
     warn(`callback ${id} may already be disposed`);
@@ -406,7 +691,7 @@ const getErrorResponse = (message, error, preparePrettyError, logError) => {
   const errorProperty = getErrorProperty(error, prettyError);
   return create$1(message, errorProperty);
 };
-const create = (message, result) => {
+const create$5 = (message, result) => {
   return {
     jsonrpc: Two,
     id: message.id,
@@ -415,7 +700,7 @@ const create = (message, result) => {
 };
 const getSuccessResponse = (message, result) => {
   const resultProperty = result ?? null;
-  return create(message, resultProperty);
+  return create$5(message, resultProperty);
 };
 const getResponse = async (message, ipc, execute, preparePrettyError, logError, requiresSocket) => {
   try {
@@ -491,484 +776,410 @@ const handleJsonRpcMessage = async (...args) => {
   }
   throw new JsonRpcError('unexpected message');
 };
-
-const requiresSocket = () => {
-  return false;
-};
-const preparePrettyError = error => {
-  return error;
-};
-const logError = error => {
-  console.error(error);
-};
-const handleMessage = event => {
-  return handleJsonRpcMessage(event.target, event.data, execute, resolve, preparePrettyError, logError, requiresSocket);
-};
-
-const handleIpc = ipc => {
-  ipc.addEventListener('message', handleMessage);
-};
-
-const MessagePort$1 = 1;
-const ModuleWorker = 2;
-const ReferencePort = 3;
-const ModuleWorkerAndMessagePort = 8;
-const Auto = () => {
-  if (globalThis.acceptPort) {
-    return MessagePort$1;
-  }
-  if (globalThis.acceptReferencePort) {
-    return ReferencePort;
-  }
-  return ModuleWorkerAndMessagePort;
-};
-
-const normalizeLine = line => {
-  if (line.startsWith('Error: ')) {
-    return line.slice('Error: '.length);
-  }
-  if (line.startsWith('VError: ')) {
-    return line.slice('VError: '.length);
+const invokeHelper = async (ipc, method, params, useSendAndTransfer) => {
+  const {
+    message,
+    promise
+  } = create$2(method, params);
+  if (useSendAndTransfer && ipc.sendAndTransfer) {
+    ipc.sendAndTransfer(message);
+  } else {
+    ipc.send(message);
   }
-  return line;
+  const responseMessage = await promise;
+  return unwrapJsonRpcResult(responseMessage);
 };
-const getCombinedMessage = (error, message) => {
-  const stringifiedError = normalizeLine(`${error}`);
-  if (message) {
-    return `${message}: ${stringifiedError}`;
-  }
-  return stringifiedError;
+const send = (transport, method, ...params) => {
+  const message = create$4(method, params);
+  transport.send(message);
 };
-const NewLine$2 = '\n';
-const getNewLineIndex = (string, startIndex = undefined) => {
-  return string.indexOf(NewLine$2, startIndex);
+const invoke = (ipc, method, ...params) => {
+  return invokeHelper(ipc, method, params, false);
 };
-const mergeStacks = (parent, child) => {
-  if (!child) {
-    return parent;
-  }
-  const parentNewLineIndex = getNewLineIndex(parent);
-  const childNewLineIndex = getNewLineIndex(child);
-  if (childNewLineIndex === -1) {
-    return parent;
-  }
-  const parentFirstLine = parent.slice(0, parentNewLineIndex);
-  const childRest = child.slice(childNewLineIndex);
-  const childFirstLine = normalizeLine(child.slice(0, childNewLineIndex));
-  if (parentFirstLine.includes(childFirstLine)) {
-    return parentFirstLine + childRest;
-  }
-  return child;
+const invokeAndTransfer = (ipc, method, ...params) => {
+  return invokeHelper(ipc, method, params, true);
 };
-class VError extends Error {
-  constructor(error, message) {
-    const combinedMessage = getCombinedMessage(error, message);
-    super(combinedMessage);
-    this.name = 'VError';
-    if (error instanceof Error) {
-      this.stack = mergeStacks(this.stack, error.stack);
-    }
-    if (error.codeFrame) {
-      // @ts-ignore
-      this.codeFrame = error.codeFrame;
-    }
-    if (error.code) {
-      // @ts-ignore
-      this.code = error.code;
-    }
-  }
-}
 
-const isMessagePort = value => {
-  return value && value instanceof MessagePort;
-};
-const isMessagePortMain = value => {
-  return value && value.constructor && value.constructor.name === 'MessagePortMain';
-};
-const isOffscreenCanvas = value => {
-  return typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas;
-};
-const isInstanceOf = (value, constructorName) => {
-  return value?.constructor?.name === constructorName;
-};
-const isSocket = value => {
-  return isInstanceOf(value, 'Socket');
-};
-const transferrables = [isMessagePort, isMessagePortMain, isOffscreenCanvas, isSocket];
-const isTransferrable = value => {
-  for (const fn of transferrables) {
-    if (fn(value)) {
-      return true;
-    }
-  }
-  return false;
-};
-const walkValue = (value, transferrables, isTransferrable) => {
-  if (!value) {
-    return;
-  }
-  if (isTransferrable(value)) {
-    transferrables.push(value);
-    return;
-  }
-  if (Array.isArray(value)) {
-    for (const item of value) {
-      walkValue(item, transferrables, isTransferrable);
-    }
-    return;
-  }
-  if (typeof value === 'object') {
-    for (const property of Object.values(value)) {
-      walkValue(property, transferrables, isTransferrable);
-    }
-    return;
-  }
-};
-const getTransferrables = value => {
-  const transferrables = [];
-  walkValue(value, transferrables, isTransferrable);
-  return transferrables;
+const commands = Object.create(null);
+const register = commandMap => {
+  Object.assign(commands, commandMap);
 };
-const attachEvents = that => {
-  const handleMessage = (...args) => {
-    const data = that.getData(...args);
-    that.dispatchEvent(new MessageEvent('message', {
-      data
-    }));
-  };
-  that.onMessage(handleMessage);
-  const handleClose = event => {
-    that.dispatchEvent(new Event('close'));
-  };
-  that.onClose(handleClose);
+const getCommand = key => {
+  return commands[key];
 };
-class Ipc extends EventTarget {
-  constructor(rawIpc) {
-    super();
-    this._rawIpc = rawIpc;
-    attachEvents(this);
+const execute = (command, ...args) => {
+  const fn = getCommand(command);
+  if (!fn) {
+    throw new Error(`command not found ${command}`);
   }
-}
-const E_INCOMPATIBLE_NATIVE_MODULE = 'E_INCOMPATIBLE_NATIVE_MODULE';
-const E_MODULES_NOT_SUPPORTED_IN_ELECTRON = 'E_MODULES_NOT_SUPPORTED_IN_ELECTRON';
-const ERR_MODULE_NOT_FOUND = 'ERR_MODULE_NOT_FOUND';
-const NewLine$1 = '\n';
-const joinLines = lines => {
-  return lines.join(NewLine$1);
-};
-const RE_AT = /^\s+at/;
-const RE_AT_PROMISE_INDEX = /^\s*at async Promise.all \(index \d+\)$/;
-const isNormalStackLine = line => {
-  return RE_AT.test(line) && !RE_AT_PROMISE_INDEX.test(line);
+  return fn(...args);
 };
-const getDetails = lines => {
-  const index = lines.findIndex(isNormalStackLine);
-  if (index === -1) {
-    return {
-      actualMessage: joinLines(lines),
-      rest: []
-    };
-  }
-  let lastIndex = index - 1;
-  while (++lastIndex < lines.length) {
-    if (!isNormalStackLine(lines[lastIndex])) {
-      break;
+
+const createRpc = ipc => {
+  const rpc = {
+    // @ts-ignore
+    ipc,
+    /**
+     * @deprecated
+     */
+    send(method, ...params) {
+      send(ipc, method, ...params);
+    },
+    invoke(method, ...params) {
+      return invoke(ipc, method, ...params);
+    },
+    invokeAndTransfer(method, ...params) {
+      return invokeAndTransfer(ipc, method, ...params);
     }
-  }
-  return {
-    actualMessage: lines[index - 1],
-    rest: lines.slice(index, lastIndex)
   };
+  return rpc;
 };
-const splitLines$1 = lines => {
-  return lines.split(NewLine$1);
-};
-const RE_MESSAGE_CODE_BLOCK_START = /^Error: The module '.*'$/;
-const RE_MESSAGE_CODE_BLOCK_END = /^\s* at/;
-const isMessageCodeBlockStartIndex = line => {
-  return RE_MESSAGE_CODE_BLOCK_START.test(line);
-};
-const isMessageCodeBlockEndIndex = line => {
-  return RE_MESSAGE_CODE_BLOCK_END.test(line);
+const requiresSocket = () => {
+  return false;
 };
-const getMessageCodeBlock = stderr => {
-  const lines = splitLines$1(stderr);
-  const startIndex = lines.findIndex(isMessageCodeBlockStartIndex);
-  const endIndex = startIndex + lines.slice(startIndex).findIndex(isMessageCodeBlockEndIndex, startIndex);
-  const relevantLines = lines.slice(startIndex, endIndex);
-  const relevantMessage = relevantLines.join(' ').slice('Error: '.length);
-  return relevantMessage;
+const preparePrettyError = error => {
+  return error;
 };
-const isModuleNotFoundMessage = line => {
-  return line.includes('[ERR_MODULE_NOT_FOUND]');
+const logError = () => {
+  // handled by renderer worker
 };
-const getModuleNotFoundError = stderr => {
-  const lines = splitLines$1(stderr);
-  const messageIndex = lines.findIndex(isModuleNotFoundMessage);
-  const message = lines[messageIndex];
-  return {
-    message,
-    code: ERR_MODULE_NOT_FOUND
-  };
+const handleMessage = event => {
+  const actualRequiresSocket = event?.target?.requiresSocket || requiresSocket;
+  const actualExecute = event?.target?.execute || execute;
+  return handleJsonRpcMessage(event.target, event.data, actualExecute, resolve, preparePrettyError, logError, actualRequiresSocket);
 };
-const isModuleNotFoundError = stderr => {
-  if (!stderr) {
-    return false;
+const handleIpc = ipc => {
+  if ('addEventListener' in ipc) {
+    ipc.addEventListener('message', handleMessage);
+  } else if ('on' in ipc) {
+    // deprecated
+    ipc.on('message', handleMessage);
   }
-  return stderr.includes('ERR_MODULE_NOT_FOUND');
 };
-const isModulesSyntaxError = stderr => {
-  if (!stderr) {
-    return false;
+const listen$1 = async (module, options) => {
+  const rawIpc = await module.listen(options);
+  if (module.signal) {
+    module.signal(rawIpc);
   }
-  return stderr.includes('SyntaxError: Cannot use import statement outside a module');
+  const ipc = module.wrap(rawIpc);
+  return ipc;
 };
-const RE_NATIVE_MODULE_ERROR = /^innerError Error: Cannot find module '.*.node'/;
-const RE_NATIVE_MODULE_ERROR_2 = /was compiled against a different Node.js version/;
-const isUnhelpfulNativeModuleError = stderr => {
-  return RE_NATIVE_MODULE_ERROR.test(stderr) && RE_NATIVE_MODULE_ERROR_2.test(stderr);
+const create$6 = async ({
+  commandMap,
+  messagePort
+}) => {
+  // TODO create a commandMap per rpc instance
+  register(commandMap);
+  const ipc = await listen$1(IpcChildWithMessagePort$1, {
+    port: messagePort
+  });
+  handleIpc(ipc);
+  const rpc = createRpc(ipc);
+  return rpc;
 };
-const getNativeModuleErrorMessage = stderr => {
-  const message = getMessageCodeBlock(stderr);
-  return {
-    message: `Incompatible native node module: ${message}`,
-    code: E_INCOMPATIBLE_NATIVE_MODULE
-  };
+const MessagePortRpcClient = {
+  __proto__: null,
+  create: create$6
 };
-const getModuleSyntaxError = () => {
-  return {
-    message: `ES Modules are not supported in electron`,
-    code: E_MODULES_NOT_SUPPORTED_IN_ELECTRON
-  };
+const create = async ({
+  commandMap
+}) => {
+  // TODO create a commandMap per rpc instance
+  register(commandMap);
+  const ipc = await listen$1(IpcChildWithModuleWorkerAndMessagePort$1);
+  handleIpc(ipc);
+  const rpc = createRpc(ipc);
+  return rpc;
 };
-const getHelpfulChildProcessError = (stdout, stderr) => {
-  if (isUnhelpfulNativeModuleError(stderr)) {
-    return getNativeModuleErrorMessage(stderr);
-  }
-  if (isModulesSyntaxError(stderr)) {
-    return getModuleSyntaxError();
+const WebWorkerRpcClient = {
+  __proto__: null,
+  create
+};
+
+const GetTokensViewport = 'GetTokensViewport.getTokensViewport';
+const HandleMessagePort = 'HandleMessagePort.handleMessagePort';
+const LoadTokenizer = 'Tokenizer.load';
+const SetLines = 'TextDocument.setLines';
+const TokenizeCodeBlock = 'Tokenizer.tokenizeCodeBlock';
+const TokenizeIncremental = 'TokenizeIncremental.tokenizeIncremental';
+
+const deepCopy = value => {
+  return structuredClone(value);
+};
+
+const getInitialLineState = initialLineState => {
+  return deepCopy(initialLineState);
+};
+
+const state = {
+  warned: []
+};
+const flattenTokensArray = tokens => {
+  const flattened = [];
+  for (const token of tokens) {
+    object(token);
+    flattened.push(token.type, token.length);
   }
-  if (isModuleNotFoundError(stderr)) {
-    return getModuleNotFoundError(stderr);
+  return flattened;
+};
+const warnDeprecatedArrayReturn = (languageId, fn) => {
+  if (state.warned.includes(fn)) {
+    return;
   }
-  const lines = splitLines$1(stderr);
-  const {
-    actualMessage,
-    rest
-  } = getDetails(lines);
-  return {
-    message: actualMessage,
-    code: '',
-    stack: rest
-  };
+  state.warned.push(fn);
+  console.warn(`tokenizers without hasArrayReturn=false are deprecated (language ${languageId})`);
 };
-class IpcError extends VError {
-  // @ts-ignore
-  constructor(betterMessage, stdout = '', stderr = '') {
-    if (stdout || stderr) {
-      // @ts-ignore
-      const {
-        message,
-        code,
-        stack
-      } = getHelpfulChildProcessError(stdout, stderr);
-      const cause = new Error(message);
-      // @ts-ignore
-      cause.code = code;
-      cause.stack = stack;
-      super(cause, betterMessage);
-    } else {
-      super(betterMessage);
+/**
+ *
+ * @param {(line:string, lineState)=>any} tokenizeLine
+ * @param {string} line
+ * @param {any} lineStateAtStart
+ * @param {boolean} hasArrayReturn
+ * @returns
+ */
+const safeTokenizeLine = (languageId, tokenizeLine, line, lineStateAtStart, hasArrayReturn) => {
+  try {
+    const lineState = tokenizeLine(line, lineStateAtStart);
+    if (!lineState || !lineState.tokens || !lineState.state) {
+      throw new Error('invalid tokenization result');
     }
-    // @ts-ignore
-    this.name = 'IpcError';
-    // @ts-ignore
-    this.stdout = stdout;
-    // @ts-ignore
-    this.stderr = stderr;
+    if (!hasArrayReturn) {
+      warnDeprecatedArrayReturn(languageId, tokenizeLine);
+      // workaround for old tokenizers
+      lineState.tokens = flattenTokensArray(lineState.tokens);
+    }
+    return lineState;
+  } catch (error) {
+    console.error(error);
+    return {
+      tokens: [/* type */0, /* length */line.length],
+      lineState: lineStateAtStart
+    };
   }
-}
-const readyMessage = 'ready';
-const getData$2 = event => {
-  return event.data;
 };
-const listen$8 = ({
-  port
-}) => {
-  return port;
+
+const lineMap = Object.create(null);
+const lineCaches = Object.create(null);
+const invalidStartIndices = Object.create(null);
+const languageIds = Object.create(null);
+const setLines = (editorId, lines) => {
+  lineMap[editorId] = lines;
+  lineCaches[editorId] = [];
+  invalidStartIndices[editorId] = 0;
 };
-const signal$9 = port => {
-  port.postMessage(readyMessage);
+const getLines = editorId => {
+  return lineMap[editorId];
 };
-class IpcChildWithMessagePort extends Ipc {
-  getData(event) {
-    return getData$2(event);
-  }
-  send(message) {
-    this._rawIpc.postMessage(message);
-  }
-  sendAndTransfer(message) {
-    const transfer = getTransferrables(message);
-    this._rawIpc.postMessage(message, transfer);
-  }
-  dispose() {
-    // ignore
-  }
-  onClose(callback) {
-    // ignore
-  }
-  onMessage(callback) {
-    this._rawIpc.addEventListener('message', callback);
-    this._rawIpc.start();
-  }
-}
-const wrap$g = port => {
-  return new IpcChildWithMessagePort(port);
+const getLineCache = editorId => {
+  return lineCaches[editorId];
 };
-const IpcChildWithMessagePort$1 = {
-  __proto__: null,
-  listen: listen$8,
-  signal: signal$9,
-  wrap: wrap$g
+const getInvalidStartIndex = editorId => {
+  return invalidStartIndices[editorId];
 };
-const listen$7 = () => {
-  // @ts-ignore
-  if (typeof WorkerGlobalScope === 'undefined') {
-    throw new TypeError('module is not in web worker scope');
+const resetInvalidStartIndex = languageId => {
+  for (const [key, value] of Object.entries(languageIds)) {
+    if (value === languageId) {
+      invalidStartIndices[key] = 0;
+    }
   }
-  return globalThis;
 };
-const signal$8 = global => {
-  global.postMessage(readyMessage);
+const setInvalidStartIndex = (editorId, index) => {
+  invalidStartIndices[editorId] = index;
 };
-class IpcChildWithModuleWorker extends Ipc {
-  getData(event) {
-    return getData$2(event);
-  }
-  send(message) {
-    // @ts-ignore
-    this._rawIpc.postMessage(message);
-  }
-  sendAndTransfer(message) {
-    const transfer = getTransferrables(message);
-    // @ts-ignore
-    this._rawIpc.postMessage(message, transfer);
-  }
-  dispose() {
-    // ignore
-  }
-  onClose(callback) {
-    // ignore
-  }
-  onMessage(callback) {
-    this._rawIpc.addEventListener('message', callback);
-  }
-}
-const wrap$f = global => {
-  return new IpcChildWithModuleWorker(global);
+const setLanguageId = (editorId, languageId) => {
+  languageIds[editorId] = languageId;
+};
+
+/**
+ * @enum number
+ */
+const State = {
+  TopLevelContent: 1
 };
-const IpcChildWithModuleWorker$1 = {
-  __proto__: null,
-  listen: listen$7,
-  signal: signal$8,
-  wrap: wrap$f
+
+/**
+ * @enum number
+ */
+const TokenType = {
+  Text: 1
 };
-const withResolvers = () => {
-  let _resolve;
-  const promise = new Promise(resolve => {
-    _resolve = resolve;
-  });
+const TokenMap = {
+  [TokenType.Text]: 'Text'
+};
+const initialLineState = {
+  state: State.TopLevelContent
+};
+const hasArrayReturn = true;
+const tokenizeLine = (line, lineState) => {
   return {
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    resolve: _resolve,
-    promise
+    tokens: [TokenType.Text, line.length],
+    state: lineState.state
   };
 };
-const waitForFirstMessage = async port => {
-  const {
-    resolve,
-    promise
-  } = withResolvers();
-  port.addEventListener('message', resolve, {
-    once: true
-  });
-  const event = await promise;
-  // @ts-ignore
-  return event.data;
+
+const TokenizePlainText = {
+  __proto__: null,
+  State,
+  TokenMap,
+  TokenType,
+  hasArrayReturn,
+  initialLineState,
+  tokenizeLine
 };
-const listen$6 = async () => {
-  const parentIpcRaw = listen$7();
-  signal$8(parentIpcRaw);
-  const parentIpc = wrap$f(parentIpcRaw);
-  const firstMessage = await waitForFirstMessage(parentIpc);
-  if (firstMessage.method !== 'initialize') {
-    throw new IpcError('unexpected first message');
-  }
-  const type = firstMessage.params[0];
-  if (type === 'message-port') {
-    parentIpc.send({
-      jsonrpc: '2.0',
-      id: firstMessage.id,
-      result: null
-    });
-    parentIpc.dispose();
-    const port = firstMessage.params[1];
-    return port;
-  }
-  return globalThis;
+
+const tokenizers = Object.create(null);
+const set$1 = (id, value) => {
+  tokenizers[id] = value;
+  resetInvalidStartIndex(id);
 };
-class IpcChildWithModuleWorkerAndMessagePort extends Ipc {
-  getData(event) {
-    return getData$2(event);
-  }
-  send(message) {
-    this._rawIpc.postMessage(message);
-  }
-  sendAndTransfer(message) {
-    const transfer = getTransferrables(message);
-    this._rawIpc.postMessage(message, transfer);
+const get = id => {
+  return tokenizers[id] || TokenizePlainText;
+};
+
+// TODO loadTokenizer should be invoked from renderer worker
+const loadTokenizer = async (languageId, tokenizePath) => {
+  if (!tokenizePath) {
+    return;
   }
-  dispose() {
-    if (this._rawIpc.close) {
-      this._rawIpc.close();
+  try {
+    // TODO check that tokenizer is valid
+    // 1. tokenizeLine should be of type function
+    // 2. getTokenClass should be of type function
+    const tokenizer = await import(tokenizePath);
+    if (typeof tokenizer.tokenizeLine !== 'function') {
+      console.warn(`tokenizer.tokenizeLine should be a function in "${tokenizePath}"`);
+      return;
     }
+    if (!tokenizer.TokenMap || typeof tokenizer.TokenMap !== 'object' || Array.isArray(tokenizer.TokenMap)) {
+      console.warn(`tokenizer.TokenMap should be an object in "${tokenizePath}"`);
+      return;
+    }
+    set$1(languageId, tokenizer);
+    return tokenizer.TokenMap;
+  } catch (error) {
+    // TODO better error handling
+    console.error(error);
+    return;
   }
-  onClose(callback) {
-    // ignore
-  }
-  onMessage(callback) {
-    this._rawIpc.addEventListener('message', callback);
-    this._rawIpc.start();
-  }
-}
-const wrap$e = port => {
-  return new IpcChildWithModuleWorkerAndMessagePort(port);
 };
-const IpcChildWithModuleWorkerAndMessagePort$1 = {
-  __proto__: null,
-  listen: listen$6,
-  wrap: wrap$e
+const getTokenizer = languageId => {
+  if (get(languageId)) {
+    return get(languageId);
+  }
+  return TokenizePlainText;
 };
 
-const getModule = method => {
-  switch (method) {
-    case ModuleWorker:
-      return IpcChildWithModuleWorker$1;
-    case ModuleWorkerAndMessagePort:
-      return IpcChildWithModuleWorkerAndMessagePort$1;
-    case MessagePort$1:
-      return IpcChildWithMessagePort$1;
-    default:
-      throw new Error('unexpected ipc type');
+const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
+  const tokenizersToLoad = [];
+  const embeddedResults = [];
+  let topContext;
+  for (const index of linesWithEmbed) {
+    const result = lineCache[index + 1];
+    const line = lines[index];
+    if (result.embeddedLanguage) {
+      const {
+        embeddedLanguage,
+        embeddedLanguageStart,
+        embeddedLanguageEnd
+      } = result;
+      const embeddedTokenizer = getTokenizer(embeddedLanguage);
+      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
+        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
+        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
+        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
+        topContext = embedResult;
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push({
+          result: embedResult,
+          TokenMap: embeddedTokenizer.TokenMap,
+          isFull
+        });
+      } else if (line.length === 0) {
+        const embedResult = {
+          tokens: []
+        };
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push({
+          result: embedResult,
+          isFull: true,
+          TokenMap: []
+        });
+      } else {
+        tokenizersToLoad.push(embeddedLanguage);
+        embeddedResults.push({
+          result: {},
+          isFull: false,
+          TokenMap: []
+        });
+        topContext = undefined;
+      }
+    } else {
+      topContext = undefined;
+    }
   }
+  return {
+    tokenizersToLoad,
+    embeddedResults
+  };
+};
+const getTokenizeEndIndex = (invalidStartIndex, endLineIndex, tokenizeStartIndex) => {
+  return invalidStartIndex < endLineIndex ? endLineIndex : tokenizeStartIndex;
 };
 
-const handleMessagePort = port => {
-  const module = getModule(MessagePort$1);
-  // @ts-ignore
-  const ipc = module.wrap(port);
-  handleIpc(ipc);
-  ipc.send('ready');
+// TODO only send changed lines to renderer process instead of all lines in viewport
+const getTokensViewport = (editor, startLineIndex, endLineIndex, hasLinesToSend, id, linesToSend) => {
+  if (hasLinesToSend) {
+    setLines(id, linesToSend);
+  }
+  const {
+    languageId
+  } = editor;
+  setLanguageId(id, languageId);
+  const lines = getLines(id);
+  const lineCache = getLineCache(id);
+  const invalidStartIndex = getInvalidStartIndex(id);
+  const tokenizer = get(languageId);
+  const {
+    hasArrayReturn,
+    tokenizeLine,
+    initialLineState
+  } = tokenizer;
+  const tokenizeStartIndex = invalidStartIndex;
+  const tokenizeEndIndex = getTokenizeEndIndex(invalidStartIndex, endLineIndex, tokenizeStartIndex);
+  const tokenizersToLoad = [];
+  const embeddedResults = [];
+  const linesWithEmbed = [];
+  for (let i = tokenizeStartIndex; i < tokenizeEndIndex; i++) {
+    const lineState = i === 0 ? getInitialLineState(initialLineState) : lineCache[i];
+    const line = lines[i];
+    const result = safeTokenizeLine(languageId, tokenizeLine, line, lineState, hasArrayReturn);
+    // TODO if lineCacheEnd matches the one before, skip tokenizing lines after
+    lineCache[i + 1] = result;
+    if (result.embeddedLanguage) {
+      result.embeddedResultIndex = linesWithEmbed.length;
+      linesWithEmbed.push(i);
+    }
+  }
+  const visibleLines = lineCache.slice(startLineIndex + 1, endLineIndex + 1);
+  if (linesWithEmbed.length > 0) {
+    const {
+      tokenizersToLoad,
+      embeddedResults
+    } = getTokensViewportEmbedded(languageId, lines, lineCache, linesWithEmbed);
+    // TODO support lineCache with embedded content
+    setInvalidStartIndex(id, 0);
+    return {
+      tokens: visibleLines,
+      tokenizersToLoad,
+      embeddedResults
+    };
+  }
+  setInvalidStartIndex(id, Math.max(invalidStartIndex, tokenizeEndIndex));
+  return {
+    tokens: visibleLines,
+    tokenizersToLoad,
+    embeddedResults
+  };
 };
 
 const getLineInfo = (line, tokens, TokenMap) => {
@@ -1029,7 +1240,7 @@ const tokenizeCodeBlock = async (codeBlock, languageId, tokenizerPath) => {
 
 const tokenizeIncremental = (id, languageId, oldLine, newLine, rowIndex, minLineY) => {
   const lineCache = getLineCache(id);
-  const tokenizer = get$1(languageId);
+  const tokenizer = get(languageId);
   const initialLineState = lineCache[rowIndex] || getInitialLineState(tokenizer.initialLineState);
   const {
     tokens: oldTokens
@@ -1066,37 +1277,38 @@ const tokenizeIncremental = (id, languageId, oldLine, newLine, rowIndex, minLine
   return undefined;
 };
 
-const commandMap = {
+const commandMap$1 = {
   [GetTokensViewport]: getTokensViewport,
-  [HandleMessagePort]: handleMessagePort,
   [LoadTokenizer]: loadTokenizer,
   [SetLines]: setLines,
   [TokenizeCodeBlock]: tokenizeCodeBlock,
   [TokenizeIncremental]: tokenizeIncremental
 };
 
-const listen$1 = async ({
-  method
-}) => {
-  const module = await getModule(method);
-  // @ts-ignore
-  const rawIpc = await module.listen();
-  // @ts-ignore
-  if (module.signal) {
-    // @ts-ignore
-    module.signal(rawIpc);
-  }
-  // @ts-ignore
-  const ipc = module.wrap(rawIpc);
-  return ipc;
+const handleMessagePort = async port => {
+  await MessagePortRpcClient.create({
+    commandMap: commandMap$1,
+    messagePort: port
+  });
+};
+
+const commandMap = {
+  ...commandMap$1,
+  [HandleMessagePort]: handleMessagePort
+};
+
+const RendererWorker = 1;
+
+const rpcs = Object.create(null);
+const set = (id, rpc) => {
+  rpcs[id] = rpc;
 };
 
 const listen = async () => {
-  register(commandMap);
-  const ipc = await listen$1({
-    method: Auto()
+  const rpc = await WebWorkerRpcClient.create({
+    commandMap: commandMap
   });
-  handleIpc(ipc);
+  set(RendererWorker, rpc);
 };
 
 const main = async () => {