@lvce-editor/editor-worker 18.24.0 → 18.26.0

package/dist/editorWorkerMain.js

@@ -1537,12 +1537,12 @@ const launchColorPickerWorker = async () => {
   return launchWorker(name, url);
 };
 
-let workerPromise$3;
+const state$f = {};
 const getOrCreate$3 = () => {
-  if (!workerPromise$3) {
-    workerPromise$3 = launchColorPickerWorker();
+  if (!state$f.workerPromise) {
+    state$f.workerPromise = launchColorPickerWorker();
   }
-  return workerPromise$3;
+  return state$f.workerPromise;
 };
 const invoke$9 = async (method, ...params) => {
   const worker = await getOrCreate$3();
@@ -1845,7 +1845,7 @@ const getInitialLineState = initialLineState => {
   return deepCopy(initialLineState);
 };
 
-const state$8 = {
+const state$e = {
   warned: []
 };
 const flattenTokensArray = tokens => {
@@ -1857,10 +1857,10 @@ const flattenTokensArray = tokens => {
   return flattened;
 };
 const warnDeprecatedArrayReturn = (languageId, fn) => {
-  if (state$8.warned.includes(fn)) {
+  if (state$e.warned.includes(fn)) {
     return;
   }
-  state$8.warned.push(fn);
+  state$e.warned.push(fn);
   console.warn(`tokenizers without hasArrayReturn=false are deprecated (language ${languageId})`);
 };
 const safeTokenizeLine = (languageId, tokenizeLine, line, lineStateAtStart, hasArrayReturn) => {
@@ -1921,12 +1921,14 @@ const TokenizePlainText = {
   tokenizeLine
 };
 
-let enabled$1 = false;
+const state$d = {
+  enabled: false
+};
 const setEnabled$1 = value => {
-  enabled$1 = value;
+  state$d.enabled = value;
 };
 const getEnabled$1 = () => {
-  return enabled$1;
+  return state$d.enabled;
 };
 
 const {
@@ -1934,21 +1936,35 @@ const {
   set: set$5
 } = SyntaxHighlightingWorker;
 
-const state$7 = {
+const state$c = {
   pending: Object.create(null),
+  tokenizePaths: Object.create(null),
   tokenizers: Object.create(null)
 };
 const has = languageId => {
-  return Object.hasOwn(state$7.tokenizers, languageId);
+  return Object.hasOwn(state$c.tokenizers, languageId);
 };
 const set$4 = (languageId, tokenizer) => {
-  state$7.tokenizers[languageId] = tokenizer;
+  state$c.tokenizers[languageId] = tokenizer;
 };
 const get$4 = languageId => {
-  return state$7.tokenizers[languageId];
+  return state$c.tokenizers[languageId];
+};
+const setTokenizePath = (languageId, tokenizePath) => {
+  state$c.tokenizePaths[languageId] = tokenizePath;
+};
+const getTokenizePath$1 = languageId => {
+  return state$c.tokenizePaths[languageId] || '';
+};
+const setTokenizePaths = languages => {
+  for (const language of languages) {
+    if (language && language.id && language.tokenize) {
+      setTokenizePath(language.id, language.tokenize);
+    }
+  }
 };
 const isPending = languageId => {
-  return Object.hasOwn(state$7.pending, languageId);
+  return Object.hasOwn(state$c.pending, languageId);
 };
 
 const tokenMaps = Object.create(null);
@@ -1964,6 +1980,7 @@ const loadTokenizer = async (languageId, tokenizePath) => {
   if (!tokenizePath) {
     return;
   }
+  setTokenizePath(languageId, tokenizePath);
   if (getEnabled$1()) {
     // @ts-ignore
     const tokenMap = await invoke$8('Tokenizer.load', languageId, tokenizePath);
@@ -2008,10 +2025,37 @@ const get$2 = id => {
   return tokenizers[id] || TokenizePlainText;
 };
 
+const getEmbeddedTokenization = (langageId, line, embeddedLanguage, embeddedLanguageStart, embeddedLanguageEnd, topContexts, tokenizersToLoad) => {
+  const embeddedTokenizer = getTokenizer(embeddedLanguage);
+  if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
+    const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
+    const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
+    const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContexts[embeddedLanguage] || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
+    topContexts[embeddedLanguage] = embedResult;
+    if (embedResult.embeddedLanguage) {
+      const nested = getEmbeddedTokenization(langageId, partialLine, embedResult.embeddedLanguage, embedResult.embeddedLanguageStart, embedResult.embeddedLanguageEnd, topContexts, tokenizersToLoad);
+      if (nested?.isFull) {
+        return nested;
+      }
+    }
+    return {
+      isFull,
+      result: embedResult,
+      TokenMap: embeddedTokenizer.TokenMap
+    };
+  }
+  tokenizersToLoad.push(embeddedLanguage);
+  topContexts[embeddedLanguage] = undefined;
+  return {
+    isFull: false,
+    result: {},
+    TokenMap: []
+  };
+};
 const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
   const tokenizersToLoad = [];
   const embeddedResults = [];
-  let topContext;
+  const topContexts = Object.create(null);
   for (const index of linesWithEmbed) {
     const result = lineCache[index + 1];
     const line = lines[index];
@@ -2021,19 +2065,7 @@ const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed)
         embeddedLanguageEnd,
         embeddedLanguageStart
       } = result;
-      const embeddedTokenizer = getTokenizer(embeddedLanguage);
-      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
-        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
-        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
-        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
-        topContext = embedResult;
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          isFull,
-          result: embedResult,
-          TokenMap: embeddedTokenizer.TokenMap
-        });
-      } else if (line.length === 0) {
+      if (line.length === 0) {
         const embedResult = {
           tokens: []
         };
@@ -2044,16 +2076,13 @@ const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed)
           TokenMap: []
         });
       } else {
-        tokenizersToLoad.push(embeddedLanguage);
-        embeddedResults.push({
-          isFull: false,
-          result: {},
-          TokenMap: []
-        });
-        topContext = undefined;
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push(getEmbeddedTokenization(langageId, line, embeddedLanguage, embeddedLanguageStart, embeddedLanguageEnd, topContexts, tokenizersToLoad));
       }
     } else {
-      topContext = undefined;
+      for (const embeddedLanguage of Object.keys(topContexts)) {
+        topContexts[embeddedLanguage] = undefined;
+      }
     }
   }
   return {
@@ -2155,8 +2184,8 @@ const getTokensViewport2 = async (editor, startLineIndex, endLineIndex, syncIncr
 
 const loadTokenizers = async languageIds => {
   for (const languageId of languageIds) {
-    // @ts-ignore
-    await loadTokenizer(languageId);
+    const tokenizePath = getTokenizePath$1(languageId);
+    await loadTokenizer(languageId, tokenizePath);
   }
 };
 
@@ -2360,6 +2389,8 @@ const positionAt = (textDocument, offset) => {
   };
 };
 
+const maxTokenizerLoadPasses = 10;
+
 // const getTokensIncremental = (editor, min, max) => {
 //   const currentLength = editor.lineStateCache.length
 //   const tokens = []
@@ -2706,20 +2737,27 @@ const getVisible$1 = async (editor, syncIncremental) => {
   } = editor;
   const maxLineY = Math.min(minLineY + numberOfVisibleLines, lines.length);
   // @ts-ignore
-  const {
+  let {
     embeddedResults,
     tokenizersToLoad,
     tokens
   } = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
+  for (let i = 0; tokenizersToLoad.length > 0 && i < maxTokenizerLoadPasses; i++) {
+    await loadTokenizers(tokenizersToLoad);
+    // @ts-ignore
+    const refreshed = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
+    ({
+      embeddedResults,
+      tokenizersToLoad,
+      tokens
+    } = refreshed);
+  }
   const minLineOffset = await offsetAtSync(editor, minLineY, 0);
   const averageCharWidth = charWidth;
   const {
     differences,
     result
   } = getLineInfosViewport(editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth);
-  if (tokenizersToLoad.length > 0) {
-    loadTokenizers(tokenizersToLoad);
-  }
   return {
     differences,
     textInfos: result
@@ -2772,12 +2810,14 @@ const getNewDeltaPercent = (height, scrollBarHeight, relativeY) => {
   };
 };
 
-let enabled = false;
+const state$b = {
+  enabled: false
+};
 const setEnabled = value => {
-  enabled = value;
+  state$b.enabled = value;
 };
 const getEnabled = () => {
-  return enabled;
+  return state$b.enabled;
 };
 
 // TODO this should be in a separate scrolling module
@@ -2957,7 +2997,7 @@ const getUrlAtOffset = (editor, offset) => {
 
 const EditorChange = 1;
 
-const state$6 = Object.create(null);
+const state$a = Object.create(null);
 
 /**
  * Register a listener for a specific event type
@@ -2967,13 +3007,13 @@ const state$6 = Object.create(null);
 const registerListener$1 = (listenerType, rpcId) => {
   number(listenerType);
   number(rpcId);
-  if (!Object.hasOwn(state$6, listenerType)) {
-    state$6[listenerType] = [];
+  if (!Object.hasOwn(state$a, listenerType)) {
+    state$a[listenerType] = [];
   }
 
   // Avoid duplicate registrations
-  if (!state$6[listenerType].includes(rpcId)) {
-    state$6[listenerType].push(rpcId);
+  if (!state$a[listenerType].includes(rpcId)) {
+    state$a[listenerType].push(rpcId);
   }
 };
 
@@ -2985,10 +3025,10 @@ const registerListener$1 = (listenerType, rpcId) => {
 const unregisterListener$1 = (listenerType, rpcId) => {
   number(listenerType);
   number(rpcId);
-  if (Object.hasOwn(state$6, listenerType)) {
-    const index = state$6[listenerType].indexOf(rpcId);
+  if (Object.hasOwn(state$a, listenerType)) {
+    const index = state$a[listenerType].indexOf(rpcId);
     if (index !== -1) {
-      state$6[listenerType].splice(index, 1);
+      state$a[listenerType].splice(index, 1);
     }
   }
 };
@@ -3000,7 +3040,7 @@ const unregisterListener$1 = (listenerType, rpcId) => {
  */
 const getListeners = listenerType => {
   number(listenerType);
-  return state$6[listenerType] || [];
+  return state$a[listenerType] || [];
 };
 
 /**
@@ -3228,16 +3268,6 @@ const from = (array, getSelection) => {
   }
   return newSelections;
 };
-const push = (selections, startRowIndex, startColumnIndex, endRowIndex, endColumnIndex) => {
-  const oldLength = selections.length;
-  const newSelections = alloc(oldLength + 4);
-  newSelections.set(selections);
-  newSelections[oldLength + 1] = startRowIndex;
-  newSelections[oldLength + 2] = startColumnIndex;
-  newSelections[oldLength + 3] = endRowIndex;
-  newSelections[oldLength + 4] = endColumnIndex;
-  return newSelections;
-};
 
 // TODO maybe only accept sorted selection edits in the first place
 
@@ -4488,7 +4518,7 @@ const y = (editor, rowIndex) => {
   return offsetY;
 };
 
-const state$5 = {
+const state$9 = {
   timeout: -1
 };
 
@@ -4519,7 +4549,7 @@ const editorShowMessage = async (editor, rowIndex, columnIndex, message, isError
 
     // TODO use wrapper timing module instead of this
     // @ts-ignore
-    state$5.timeout = setTimeout(handleTimeout, 3000);
+    state$9.timeout = setTimeout(handleTimeout, 3000);
   }
   return editor;
 };
@@ -4539,8 +4569,8 @@ const showErrorMessage = async (editor, rowIndex, columnIndex, message) => {
 
 // @ts-ignore
 const editorHideMessage = async editor => {
-  clearTimeout(state$5.timeout);
-  state$5.timeout = -1;
+  clearTimeout(state$9.timeout);
+  state$9.timeout = -1;
   // await RendererProcess.invoke(/* Viewlet.send */ 'Viewlet.send', /* id */ editor.uid, /* method */ 'hideOverlayMessage')
   return editor;
 };
@@ -4656,12 +4686,12 @@ const launchRenameWorker = async () => {
   return rpc;
 };
 
-let workerPromise$2;
+const state$8 = {};
 const getOrCreate$2 = () => {
-  if (!workerPromise$2) {
-    workerPromise$2 = launchRenameWorker();
+  if (!state$8.workerPromise) {
+    state$8.workerPromise = launchRenameWorker();
   }
-  return workerPromise$2;
+  return state$8.workerPromise;
 };
 const invoke$5 = async (method, ...params) => {
   const worker = await getOrCreate$2();
@@ -4781,12 +4811,12 @@ const openColorPicker = async editor => {
   return addWidgetToEditor(ColorPicker$1, ColorPicker, editor, create$6, newStateGenerator$6);
 };
 
-const state$4 = {
+const state$7 = {
   compositionText: '',
   isComposing: false
 };
 const compositionStart = (editor, event) => {
-  state$4.isComposing = true;
+  state$7.isComposing = true;
   return editor;
 };
 const getCompositionChanges = (selections, data) => {
@@ -4796,9 +4826,9 @@ const getCompositionChanges = (selections, data) => {
     const selectionStartColumn = selections[i + 1];
     const selectionEndRow = selections[i + 2];
     const selectionEndColumn = selections[i + 3];
-    const startColumnIndex = selectionStartColumn - state$4.compositionText.length;
+    const startColumnIndex = selectionStartColumn - state$7.compositionText.length;
     changes.push({
-      deleted: [state$4.compositionText],
+      deleted: [state$7.compositionText],
       end: {
         columnIndex: selectionEndColumn,
         rowIndex: selectionEndRow
@@ -4818,7 +4848,7 @@ const compositionUpdate = (editor, data) => {
     selections
   } = editor;
   const changes = getCompositionChanges(selections, data);
-  state$4.compositionText = data;
+  state$7.compositionText = data;
   return scheduleDocumentAndCursorsSelections(editor, changes);
 };
 const compositionEnd = (editor, data) => {
@@ -4826,8 +4856,8 @@ const compositionEnd = (editor, data) => {
     selections
   } = editor;
   const changes = getCompositionChanges(selections, data);
-  state$4.isComposing = false;
-  state$4.compositionText = '';
+  state$7.isComposing = false;
+  state$7.compositionText = '';
   return scheduleDocumentAndCursorsSelections(editor, changes);
 };
 
@@ -5903,19 +5933,19 @@ const closeWidgetsMaybe = widgets => {
   return widgets.filter(isPersistentWidget);
 };
 
-const state$3 = {
+const state$6 = {
   position: {
     columnIndex: 0,
     rowIndex: 0
   }
 };
 const getPosition$1 = () => {
-  return state$3.position;
+  return state$6.position;
 };
 
 // @ts-ignore
 const setPosition$1 = position => {
-  state$3.position = position;
+  state$6.position = position;
 };
 
 const openExternal = async (url, platform) => {
@@ -6133,20 +6163,20 @@ const handleMouseDown = (state, button, altKey, ctrlKey, x, y, detail) => {
   }
 };
 
-const state$2 = {
+const state$5 = {
   editor: undefined,
   timeout: -1,
   x: 0,
   y: 0
 };
 const get = () => {
-  return state$2;
+  return state$5;
 };
 const set = (editor, timeout, x, y) => {
-  state$2.editor = editor;
-  state$2.timeout = timeout;
-  state$2.x = x;
-  state$2.y = y;
+  state$5.editor = editor;
+  state$5.timeout = timeout;
+  state$5.x = x;
+  state$5.y = y;
 };
 
 const showHover$1 = async (editor, position) => {
@@ -6287,7 +6317,7 @@ const editorHandleNativeSelectionChange = (editor, range) => {
   return scheduleSelections(editor, selections);
 };
 
-const state$1 = {
+const state$4 = {
   /**
    * @type {any}
    */
@@ -6302,33 +6332,33 @@ const state$1 = {
 
 // @ts-ignore
 const setEditor = editor => {
-  state$1.currentEditor = editor;
-  state$1.hasListener = true;
+  state$4.currentEditor = editor;
+  state$4.hasListener = true;
 };
 const clearEditor = () => {
-  state$1.currentEditor = undefined;
-  state$1.hasListener = false;
-  state$1.isSelecting = false;
+  state$4.currentEditor = undefined;
+  state$4.hasListener = false;
+  state$4.isSelecting = false;
 };
 const startSelecting = () => {
-  state$1.isSelecting = true;
+  state$4.isSelecting = true;
 };
 const isSelecting = () => {
-  return state$1.isSelecting;
+  return state$4.isSelecting;
 };
 
 // @ts-ignore
 const setPosition = position => {
-  state$1.position = position;
+  state$4.position = position;
 };
 const getEditor$1 = () => {
-  return state$1.currentEditor;
+  return state$4.currentEditor;
 };
 const getPosition = () => {
-  return state$1.position;
+  return state$4.position;
 };
 const hasListener = () => {
-  return state$1.hasListener;
+  return state$4.hasListener;
 };
 
 // @ts-ignore
@@ -6708,7 +6738,7 @@ const handleTouchEnd = (editor, touchEvent) => {
   // }
 };
 
-const state = {
+const state$3 = {
   deltaY: 0,
   touchOffsetY: 0
 };
@@ -6719,8 +6749,8 @@ const handleTouchStart = (editor, touchEvent) => {
     return;
   }
   const firstTouch = touchEvent.touches[0];
-  state.touchOffsetY = firstTouch.y;
-  state.deltaY = editor.deltaY;
+  state$3.touchOffsetY = firstTouch.y;
+  state$3.deltaY = editor.deltaY;
   // const position = EditorPosition.at(editor, firstTouch.x, firstTouch.y)
   // EditorMoveSelection.state.position = position
   // state.date = Date.now()
@@ -6761,7 +6791,7 @@ const handleTouchMove = (editor, touchEvent) => {
     return;
   }
   const firstTouch = touchEvent.touches[0];
-  const offsetY = state.deltaY + (state.touchOffsetY - firstTouch.y);
+  const offsetY = state$3.deltaY + (state$3.touchOffsetY - firstTouch.y);
   setDeltaYFixedValue(editor, offsetY);
 };
 
@@ -7783,7 +7813,13 @@ const getSelectionEditsSingleLineWord = (lines, selections) => {
   const columnIndexAfter = line.indexOf(word, lastSelectionEndColumnIndex);
   if (columnIndexAfter !== -1) {
     const columnIndexAfterEnd = columnIndexAfter + word.length;
-    const newSelections = push(selections, rowIndex, columnIndexAfter, rowIndex, columnIndexAfterEnd);
+    const newSelections = new Uint32Array(selections.length + 4);
+    newSelections.set(selections, 0);
+    const insertIndex = selections.length;
+    newSelections[insertIndex] = rowIndex;
+    newSelections[insertIndex + 1] = columnIndexAfter;
+    newSelections[insertIndex + 2] = rowIndex;
+    newSelections[insertIndex + 3] = columnIndexAfterEnd;
     return {
       revealRange: newSelections.length - 4,
       selectionEdits: newSelections
@@ -7988,6 +8024,7 @@ const setLanguageId = async (editor, languageId, tokenizePath) => {
   const {
     tokenizerId
   } = editor;
+  setTokenizePath(languageId, tokenizePath);
   // TODO move tokenizer to syntax highlighting worker
   // TODO only load tokenizer if not already loaded
   // if already loaded just set tokenizer and rerender text
@@ -8128,12 +8165,12 @@ const launchHoverWorker = async () => {
   return rpc;
 };
 
-let workerPromise$1;
+const state$2 = {};
 const getOrCreate$1 = () => {
-  if (!workerPromise$1) {
-    workerPromise$1 = launchHoverWorker();
+  if (!state$2.workerPromise) {
+    state$2.workerPromise = launchHoverWorker();
   }
-  return workerPromise$1;
+  return state$2.workerPromise;
 };
 const invoke$2 = async (method, ...params) => {
   const worker = await getOrCreate$1();
@@ -8212,12 +8249,12 @@ const launchSourceActionWorker = async () => {
   return rpc;
 };
 
-let workerPromise;
+const state$1 = {};
 const getOrCreate = () => {
-  if (!workerPromise) {
-    workerPromise = launchSourceActionWorker();
+  if (!state$1.workerPromise) {
+    state$1.workerPromise = launchSourceActionWorker();
   }
-  return workerPromise;
+  return state$1.workerPromise;
 };
 const invoke$1 = async (method, ...params) => {
   const worker = await getOrCreate();
@@ -10704,12 +10741,14 @@ const restoreWidgetState = async (keys, savedStates) => {
   return newEditors;
 };
 
-let isReloading = false;
+const state = {
+  isReloading: false
+};
 const hotReload = async () => {
-  if (isReloading) {
+  if (state.isReloading) {
     return;
   }
-  isReloading = true;
+  state.isReloading = true;
 
   // TODO use getEditors
   const keys = getKeys$2();
@@ -10723,7 +10762,7 @@ const hotReload = async () => {
   // TODO ask renderer worker to rerender all editors
   // @ts-ignore
   await invoke$b(`Editor.rerender`);
-  isReloading = false;
+  state.isReloading = false;
 };
 
 const sendMessagePortToSyntaxHighlightingWorker = async port => {
@@ -10883,6 +10922,7 @@ const loadContent = async (state, savedState) => {
   // TODO support overwriting language id by setting it explicitly or via settings
   const charWidth = await measureCharacterWidth(fontWeight, fontSize, fontFamily, letterSpacing);
   const languages = await getLanguages(platform, assetDir);
+  setTokenizePaths(languages);
   const computedLanguageId = getLanguageId$1(uri, languages);
   const tokenizePath = getTokenizePath(languages, computedLanguageId);
   await loadTokenizer(computedLanguageId, tokenizePath);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/editor-worker",
-  "version": "18.24.0",
+  "version": "18.26.0",
   "repository": {
     "type": "git",
     "url": "git@github.com:lvce-editor/editor-worker.git"