@lvce-editor/editor-worker 18.24.0 → 18.25.0

package/dist/editorWorkerMain.js

@@ -1936,6 +1936,7 @@ const {
 
 const state$7 = {
   pending: Object.create(null),
+  tokenizePaths: Object.create(null),
   tokenizers: Object.create(null)
 };
 const has = languageId => {
@@ -1947,6 +1948,19 @@ const set$4 = (languageId, tokenizer) => {
 const get$4 = languageId => {
   return state$7.tokenizers[languageId];
 };
+const setTokenizePath = (languageId, tokenizePath) => {
+  state$7.tokenizePaths[languageId] = tokenizePath;
+};
+const getTokenizePath$1 = languageId => {
+  return state$7.tokenizePaths[languageId] || '';
+};
+const setTokenizePaths = languages => {
+  for (const language of languages) {
+    if (language && language.id && language.tokenize) {
+      setTokenizePath(language.id, language.tokenize);
+    }
+  }
+};
 const isPending = languageId => {
   return Object.hasOwn(state$7.pending, languageId);
 };
@@ -1964,6 +1978,7 @@ const loadTokenizer = async (languageId, tokenizePath) => {
   if (!tokenizePath) {
     return;
   }
+  setTokenizePath(languageId, tokenizePath);
   if (getEnabled$1()) {
     // @ts-ignore
     const tokenMap = await invoke$8('Tokenizer.load', languageId, tokenizePath);
@@ -2008,10 +2023,37 @@ const get$2 = id => {
   return tokenizers[id] || TokenizePlainText;
 };
 
+const getEmbeddedTokenization = (langageId, line, embeddedLanguage, embeddedLanguageStart, embeddedLanguageEnd, topContexts, tokenizersToLoad) => {
+  const embeddedTokenizer = getTokenizer(embeddedLanguage);
+  if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
+    const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
+    const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
+    const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContexts[embeddedLanguage] || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
+    topContexts[embeddedLanguage] = embedResult;
+    if (embedResult.embeddedLanguage) {
+      const nested = getEmbeddedTokenization(langageId, partialLine, embedResult.embeddedLanguage, embedResult.embeddedLanguageStart, embedResult.embeddedLanguageEnd, topContexts, tokenizersToLoad);
+      if (nested?.isFull) {
+        return nested;
+      }
+    }
+    return {
+      isFull,
+      result: embedResult,
+      TokenMap: embeddedTokenizer.TokenMap
+    };
+  }
+  tokenizersToLoad.push(embeddedLanguage);
+  topContexts[embeddedLanguage] = undefined;
+  return {
+    isFull: false,
+    result: {},
+    TokenMap: []
+  };
+};
 const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed) => {
   const tokenizersToLoad = [];
   const embeddedResults = [];
-  let topContext;
+  const topContexts = Object.create(null);
   for (const index of linesWithEmbed) {
     const result = lineCache[index + 1];
     const line = lines[index];
@@ -2021,19 +2063,7 @@ const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed)
         embeddedLanguageEnd,
         embeddedLanguageStart
       } = result;
-      const embeddedTokenizer = getTokenizer(embeddedLanguage);
-      if (embeddedLanguageStart !== line.length && embeddedTokenizer && embeddedTokenizer !== TokenizePlainText) {
-        const isFull = embeddedLanguageStart === 0 && embeddedLanguageEnd === line.length;
-        const partialLine = line.slice(embeddedLanguageStart, embeddedLanguageEnd);
-        const embedResult = safeTokenizeLine(langageId, embeddedTokenizer.tokenizeLine, partialLine, topContext || getInitialLineState(embeddedTokenizer.initialLineState), embeddedTokenizer.hasArrayReturn);
-        topContext = embedResult;
-        result.embeddedResultIndex = embeddedResults.length;
-        embeddedResults.push({
-          isFull,
-          result: embedResult,
-          TokenMap: embeddedTokenizer.TokenMap
-        });
-      } else if (line.length === 0) {
+      if (line.length === 0) {
         const embedResult = {
           tokens: []
         };
@@ -2044,16 +2074,13 @@ const getTokensViewportEmbedded = (langageId, lines, lineCache, linesWithEmbed)
           TokenMap: []
         });
       } else {
-        tokenizersToLoad.push(embeddedLanguage);
-        embeddedResults.push({
-          isFull: false,
-          result: {},
-          TokenMap: []
-        });
-        topContext = undefined;
+        result.embeddedResultIndex = embeddedResults.length;
+        embeddedResults.push(getEmbeddedTokenization(langageId, line, embeddedLanguage, embeddedLanguageStart, embeddedLanguageEnd, topContexts, tokenizersToLoad));
       }
     } else {
-      topContext = undefined;
+      for (const embeddedLanguage of Object.keys(topContexts)) {
+        topContexts[embeddedLanguage] = undefined;
+      }
     }
   }
   return {
@@ -2155,8 +2182,8 @@ const getTokensViewport2 = async (editor, startLineIndex, endLineIndex, syncIncr
 
 const loadTokenizers = async languageIds => {
   for (const languageId of languageIds) {
-    // @ts-ignore
-    await loadTokenizer(languageId);
+    const tokenizePath = getTokenizePath$1(languageId);
+    await loadTokenizer(languageId, tokenizePath);
   }
 };
 
@@ -2360,6 +2387,8 @@ const positionAt = (textDocument, offset) => {
   };
 };
 
+const maxTokenizerLoadPasses = 10;
+
 // const getTokensIncremental = (editor, min, max) => {
 //   const currentLength = editor.lineStateCache.length
 //   const tokens = []
@@ -2706,20 +2735,27 @@ const getVisible$1 = async (editor, syncIncremental) => {
   } = editor;
   const maxLineY = Math.min(minLineY + numberOfVisibleLines, lines.length);
   // @ts-ignore
-  const {
+  let {
     embeddedResults,
     tokenizersToLoad,
     tokens
   } = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
+  for (let i = 0; tokenizersToLoad.length > 0 && i < maxTokenizerLoadPasses; i++) {
+    await loadTokenizers(tokenizersToLoad);
+    // @ts-ignore
+    const refreshed = await getTokensViewport2(editor, minLineY, maxLineY, syncIncremental);
+    ({
+      embeddedResults,
+      tokenizersToLoad,
+      tokens
+    } = refreshed);
+  }
   const minLineOffset = await offsetAtSync(editor, minLineY, 0);
   const averageCharWidth = charWidth;
   const {
     differences,
     result
   } = getLineInfosViewport(editor, tokens, embeddedResults, minLineY, maxLineY, minLineOffset, width, deltaX, averageCharWidth);
-  if (tokenizersToLoad.length > 0) {
-    loadTokenizers(tokenizersToLoad);
-  }
   return {
     differences,
     textInfos: result
@@ -7988,6 +8024,7 @@ const setLanguageId = async (editor, languageId, tokenizePath) => {
   const {
     tokenizerId
   } = editor;
+  setTokenizePath(languageId, tokenizePath);
   // TODO move tokenizer to syntax highlighting worker
   // TODO only load tokenizer if not already loaded
   // if already loaded just set tokenizer and rerender text
@@ -10883,6 +10920,7 @@ const loadContent = async (state, savedState) => {
   // TODO support overwriting language id by setting it explicitly or via settings
   const charWidth = await measureCharacterWidth(fontWeight, fontSize, fontFamily, letterSpacing);
   const languages = await getLanguages(platform, assetDir);
+  setTokenizePaths(languages);
   const computedLanguageId = getLanguageId$1(uri, languages);
   const tokenizePath = getTokenizePath(languages, computedLanguageId);
   await loadTokenizer(computedLanguageId, tokenizePath);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/editor-worker",
-  "version": "18.24.0",
+  "version": "18.25.0",
   "repository": {
     "type": "git",
     "url": "git@github.com:lvce-editor/editor-worker.git"