@lvce-editor/chat-message-parsing-worker 1.6.0 → 1.8.0

package/dist/chatMessageParsingWorkerMain.js

@@ -486,10 +486,10 @@ const execute = (command, ...args) => {
 
 const Two$1 = '2.0';
 const callbacks = Object.create(null);
-const get = id => {
+const get$1 = id => {
   return callbacks[id];
 };
-const remove = id => {
+const remove$1 = id => {
   delete callbacks[id];
 };
 class JsonRpcError extends Error {
@@ -635,14 +635,14 @@ const warn = (...args) => {
   console.warn(...args);
 };
 const resolve = (id, response) => {
-  const fn = get(id);
+  const fn = get$1(id);
   if (!fn) {
     console.log(response);
     warn(`callback ${id} may already be disposed`);
     return;
   }
   fn(response);
-  remove(id);
+  remove$1(id);
 };
 const E_COMMAND_NOT_FOUND = 'E_COMMAND_NOT_FOUND';
 const getErrorType = prettyError => {
@@ -698,7 +698,7 @@ const getErrorResponse = (id, error, preparePrettyError, logError) => {
   const errorProperty = getErrorProperty(error, prettyError);
   return create$1$1(id, errorProperty);
 };
-const create$5 = (message, result) => {
+const create$8 = (message, result) => {
   return {
     id: message.id,
     jsonrpc: Two$1,
@@ -707,7 +707,7 @@ const create$5 = (message, result) => {
 };
 const getSuccessResponse = (message, result) => {
   const resultProperty = result ?? null;
-  return create$5(message, resultProperty);
+  return create$8(message, resultProperty);
 };
 const getErrorResponseSimple = (id, error) => {
   return {
@@ -801,7 +801,7 @@ const handleJsonRpcMessage = async (...args) => {
 
 const Two = '2.0';
 
-const create$4 = (method, params) => {
+const create$7 = (method, params) => {
   return {
     jsonrpc: Two,
     method,
@@ -809,7 +809,7 @@ const create$4 = (method, params) => {
   };
 };
 
-const create$3 = (id, method, params) => {
+const create$6 = (id, method, params) => {
   const message = {
     id,
     jsonrpc: Two,
@@ -820,12 +820,12 @@ const create$3 = (id, method, params) => {
 };
 
 let id = 0;
-const create$2 = () => {
+const create$5 = () => {
   return ++id;
 };
 
 const registerPromise = map => {
-  const id = create$2();
+  const id = create$5();
   const {
     promise,
     resolve
@@ -842,7 +842,7 @@ const invokeHelper = async (callbacks, ipc, method, params, useSendAndTransfer)
     id,
     promise
   } = registerPromise(callbacks);
-  const message = create$3(id, method, params);
+  const message = create$6(id, method, params);
   if (useSendAndTransfer && ipc.sendAndTransfer) {
     ipc.sendAndTransfer(message);
   } else {
@@ -878,7 +878,7 @@ const createRpc = ipc => {
      * @deprecated
      */
     send(method, ...params) {
-      const message = create$4(method, params);
+      const message = create$7(method, params);
       ipc.send(message);
     }
   };
@@ -918,7 +918,7 @@ const listen$1 = async (module, options) => {
   return ipc;
 };
 
-const create$1 = async ({
+const create$4 = async ({
   commandMap,
   isMessagePortOpen = true,
   messagePort
@@ -936,7 +936,66 @@ const create$1 = async ({
   return rpc;
 };
 
-const create = async ({
+const create$3 = async ({
+  commandMap,
+  isMessagePortOpen,
+  send
+}) => {
+  const {
+    port1,
+    port2
+  } = new MessageChannel();
+  await send(port1);
+  return create$4({
+    commandMap,
+    isMessagePortOpen,
+    messagePort: port2
+  });
+};
+
+const createSharedLazyRpc = factory => {
+  let rpcPromise;
+  const getOrCreate = () => {
+    if (!rpcPromise) {
+      rpcPromise = factory();
+    }
+    return rpcPromise;
+  };
+  return {
+    async dispose() {
+      const rpc = await getOrCreate();
+      await rpc.dispose();
+    },
+    async invoke(method, ...params) {
+      const rpc = await getOrCreate();
+      return rpc.invoke(method, ...params);
+    },
+    async invokeAndTransfer(method, ...params) {
+      const rpc = await getOrCreate();
+      return rpc.invokeAndTransfer(method, ...params);
+    },
+    async send(method, ...params) {
+      const rpc = await getOrCreate();
+      rpc.send(method, ...params);
+    }
+  };
+};
+
+const create$2 = async ({
+  commandMap,
+  isMessagePortOpen,
+  send
+}) => {
+  return createSharedLazyRpc(() => {
+    return create$3({
+      commandMap,
+      isMessagePortOpen,
+      send
+    });
+  });
+};
+
+const create$1 = async ({
   commandMap
 }) => {
   // TODO create a commandMap per rpc instance
@@ -948,13 +1007,415 @@ const create = async ({
 };
 
 const handleMessagePort = async port => {
-  await create$1({
+  await create$4({
     commandMap: commandMap,
     isMessagePortOpen: true,
     messagePort: port
   });
 };
 
+const initialize = async (_, port) => {
+  await handleMessagePort(port);
+};
+
+const createMockRpc = ({
+  commandMap
+}) => {
+  const invocations = [];
+  const invoke = (method, ...params) => {
+    invocations.push([method, ...params]);
+    const command = commandMap[method];
+    if (!command) {
+      throw new Error(`command ${method} not found`);
+    }
+    return command(...params);
+  };
+  const mockRpc = {
+    invocations,
+    invoke,
+    invokeAndTransfer: invoke
+  };
+  return mockRpc;
+};
+
+const rpcs = Object.create(null);
+const set$1 = (id, rpc) => {
+  rpcs[id] = rpc;
+};
+const get = id => {
+  return rpcs[id];
+};
+const remove = id => {
+  delete rpcs[id];
+};
+
+/* eslint-disable  @typescript-eslint/explicit-function-return-type */
+const create = rpcId => {
+  return {
+    async dispose() {
+      const rpc = get(rpcId);
+      await rpc.dispose();
+    },
+    // @ts-ignore
+    invoke(method, ...params) {
+      const rpc = get(rpcId);
+      // @ts-ignore
+      return rpc.invoke(method, ...params);
+    },
+    // @ts-ignore
+    invokeAndTransfer(method, ...params) {
+      const rpc = get(rpcId);
+      // @ts-ignore
+      return rpc.invokeAndTransfer(method, ...params);
+    },
+    registerMockRpc(commandMap) {
+      const mockRpc = createMockRpc({
+        commandMap
+      });
+      set$1(rpcId, mockRpc);
+      // @ts-ignore
+      mockRpc[Symbol.dispose] = () => {
+        remove(rpcId);
+      };
+      // @ts-ignore
+      return mockRpc;
+    },
+    set(rpc) {
+      set$1(rpcId, rpc);
+    }
+  };
+};
+
+const {
+  invoke,
+  set
+} = create(6007);
+const getMathBlockDom = async node => {
+  return invoke('ChatMath.getMathBlockDom', node);
+};
+const getMathInlineDom = async node => {
+  return invoke('ChatMath.getMathInlineDom', node);
+};
+
+const RendererWorker = 1;
+
+const {
+  invokeAndTransfer} = create(RendererWorker);
+const sendMessagePortToChatMathWorker$1 = async (port, rpcId) => {
+  const command = 'HandleMessagePort.handleMessagePort';
+  await invokeAndTransfer('SendMessagePortToExtensionHostWorker.sendMessagePortToChatMathWorker', port, command, rpcId);
+};
+
+const parseMathInline = async children => {
+  const nextChildren = [];
+  for (const child of children) {
+    if (child.type === 'math-inline') {
+      const dom = await getMathInlineDom(child);
+      nextChildren.push({
+        dom,
+        type: 'math-inline-dom'
+      });
+      continue;
+    }
+    if (child.type === 'bold') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    if (child.type === 'italic') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    if (child.type === 'strikethrough') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    nextChildren.push(child);
+  }
+  return nextChildren;
+};
+const parseMathListItem = async item => {
+  const children = await parseMathInline(item.children);
+  if (!item.nestedItems) {
+    return {
+      ...item,
+      children
+    };
+  }
+  const nestedItems = [];
+  for (const nestedItem of item.nestedItems) {
+    nestedItems.push(await parseMathListItem(nestedItem));
+  }
+  return {
+    ...item,
+    children,
+    nestedItems
+  };
+};
+const parseMathTableCell = async cell => {
+  return {
+    ...cell,
+    children: await parseMathInline(cell.children)
+  };
+};
+const parseMathNode = async node => {
+  if (node.type === 'math-block') {
+    const dom = await getMathBlockDom(node);
+    return {
+      dom,
+      type: 'math-block-dom'
+    };
+  }
+  if (node.type === 'text' || node.type === 'heading') {
+    return {
+      ...node,
+      children: await parseMathInline(node.children)
+    };
+  }
+  if (node.type === 'blockquote') {
+    const children = [];
+    for (const child of node.children) {
+      children.push(await parseMathNode(child));
+    }
+    return {
+      ...node,
+      children
+    };
+  }
+  if (node.type === 'ordered-list' || node.type === 'unordered-list') {
+    const items = [];
+    for (const item of node.items) {
+      items.push(await parseMathListItem(item));
+    }
+    return {
+      ...node,
+      items
+    };
+  }
+  if (node.type === 'table') {
+    const headers = [];
+    for (const header of node.headers) {
+      headers.push(await parseMathTableCell(header));
+    }
+    const rows = [];
+    for (const row of node.rows) {
+      const cells = [];
+      for (const cell of row.cells) {
+        cells.push(await parseMathTableCell(cell));
+      }
+      rows.push({
+        ...row,
+        cells
+      });
+    }
+    return {
+      ...node,
+      headers,
+      rows
+    };
+  }
+  return node;
+};
+
+const TokenAttribute = 'TokenAttribute';
+const TokenComment = 'TokenComment';
+const TokenKeyword = 'TokenKeyword';
+const TokenNumber = 'TokenNumber';
+const TokenProperty = 'TokenProperty';
+const TokenString = 'TokenString';
+const TokenTag = 'TokenTag';
+const TokenValue = 'TokenValue';
+
+const jsRules = [{
+  className: TokenComment,
+  regex: /\/\/[^\n]*/
+}, {
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenString,
+  regex: /`[^`\\]*(?:\\.[^`\\]*)*`/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:const|let|var|function|return|if|else|for|while|class|new|typeof|instanceof|import|export|from|default|async|await|try|catch|finally|throw|this|true|false|null|undefined)\b/
+}];
+const tsRules = [{
+  className: TokenComment,
+  regex: /\/\/[^\n]*/
+}, {
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenString,
+  regex: /`[^`\\]*(?:\\.[^`\\]*)*`/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:abstract|any|as|asserts|async|await|boolean|class|const|constructor|declare|default|enum|export|extends|false|finally|for|from|function|get|if|implements|import|in|infer|instanceof|interface|is|keyof|let|module|namespace|never|new|null|number|object|override|private|protected|public|readonly|return|satisfies|set|static|string|super|switch|symbol|this|throw|true|try|type|typeof|undefined|unknown|var|void|while)\b/
+}];
+const htmlRules = [{
+  className: TokenComment,
+  regex: /<!--[\s\S]*?-->/
+}, {
+  className: TokenTag,
+  regex: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/
+}, {
+  className: TokenTag,
+  regex: /<[a-zA-Z][a-zA-Z0-9-]*/
+}, {
+  className: TokenTag,
+  regex: /\/?>/
+}, {
+  className: TokenString,
+  regex: /"[^"]*"/
+}, {
+  className: TokenString,
+  regex: /'[^']*'/
+}, {
+  className: TokenAttribute,
+  regex: /[a-zA-Z-]+(?=\s*=)/
+}];
+const cssRules = [{
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"]*"/
+}, {
+  className: TokenString,
+  regex: /'[^']*'/
+}, {
+  className: TokenValue,
+  regex: /#[0-9a-fA-F]{3,8}/
+}, {
+  className: TokenValue,
+  regex: /\b\d+(?:\.\d+)?(?:px|em|rem|%|vh|vw|pt|s|ms|fr)?\b/
+}, {
+  className: TokenProperty,
+  regex: /[a-zA-Z-]+(?=\s*:)/
+}];
+const pythonRules = [{
+  className: TokenComment,
+  regex: /#[^\n]*/
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:def|class|return|if|elif|else|for|while|in|import|from|as|with|try|except|finally|raise|lambda|yield|pass|break|continue|True|False|None|and|or|not|is)\b/
+}];
+const jsonRules = [{
+  className: TokenProperty,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"(?=\s*:)/
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenNumber,
+  regex: /-?\b\d+\.?\d*(?:[eE][+-]?\d+)?\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:true|false|null)\b/
+}];
+const tokenize = (code, rules) => {
+  const tokens = [];
+  let pos = 0;
+  while (pos < code.length) {
+    let matched = false;
+    for (const rule of rules) {
+      const match = rule.regex.exec(code.slice(pos));
+      if (match && match.index === 0) {
+        tokens.push({
+          className: rule.className,
+          text: match[0]
+        });
+        pos += match[0].length;
+        matched = true;
+        break;
+      }
+    }
+    if (!matched) {
+      const last = tokens.at(-1);
+      if (last && last.className === '') {
+        tokens.pop();
+        tokens.push({
+          className: '',
+          text: last.text + code[pos]
+        });
+      } else {
+        tokens.push({
+          className: '',
+          text: code[pos]
+        });
+      }
+      pos++;
+    }
+  }
+  return tokens;
+};
+const highlightCode = (code, language) => {
+  if (!language) {
+    return [{
+      className: '',
+      text: code
+    }];
+  }
+  const normalized = language.toLowerCase();
+  if (normalized === 'html') {
+    return tokenize(code, htmlRules);
+  }
+  if (normalized === 'css') {
+    return tokenize(code, cssRules);
+  }
+  if (normalized === 'js' || normalized === 'javascript') {
+    return tokenize(code, jsRules);
+  }
+  if (normalized === 'ts' || normalized === 'typescript') {
+    return tokenize(code, tsRules);
+  }
+  if (normalized === 'py' || normalized === 'python') {
+    return tokenize(code, pythonRules);
+  }
+  if (normalized === 'json') {
+    return tokenize(code, jsonRules);
+  }
+  return [{
+    className: '',
+    text: code
+  }];
+};
+
 const windowsAbsolutePathRegex = /^[a-zA-Z]:[\\/]/;
 const pathSeparatorRegex$1 = /[\\/]/;
 const isPathTraversalAttempt = path => {
@@ -1675,8 +2136,6 @@ const getEmptyTextNode = () => {
     type: 'text'
   }];
 };
-
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
 const parseBlockTokens = tokens => {
   if (tokens.length === 0) {
     return getEmptyTextNode();
@@ -1686,6 +2145,16 @@ const parseBlockTokens = tokens => {
   let listItems = [];
   let listType = '';
   let orderedListPathStack = [];
+  let canContinueOrderedListItemParagraph = false;
+  const createListItem = (text, index) => {
+    return {
+      children: parseInlineNodes(text),
+      ...(index === undefined ? {} : {
+        index
+      }),
+      type: 'list-item'
+    };
+  };
   const getListItemAtPath = (items, path) => {
     let currentItems = items;
     let currentItem;
@@ -1715,6 +2184,29 @@ const parseBlockTokens = tokens => {
     };
     return [...items.slice(0, index), nextItem, ...items.slice(index + 1)];
   };
+  const appendInlineChildrenAtPath = (items, path, children) => {
+    if (path.length === 0) {
+      return [...items];
+    }
+    const [index, ...rest] = path;
+    const current = items[index];
+    if (!current) {
+      return [...items];
+    }
+    const lineBreakNode = {
+      text: '\n',
+      type: 'text'
+    };
+    const nextChildren = [...current.children, lineBreakNode, ...children];
+    const nextItem = rest.length > 0 ? {
+      ...current,
+      nestedItems: appendInlineChildrenAtPath(current.nestedItems || [], rest, children)
+    } : {
+      ...current,
+      children: nextChildren
+    };
+    return [...items.slice(0, index), nextItem, ...items.slice(index + 1)];
+  };
   const flushParagraph = () => {
     if (paragraphLines.length === 0) {
       return;
@@ -1736,11 +2228,13 @@ const parseBlockTokens = tokens => {
     listItems = [];
     listType = '';
     orderedListPathStack = [];
+    canContinueOrderedListItemParagraph = false;
   };
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
     if (token.type === 'blank-line') {
       flushParagraph();
+      canContinueOrderedListItemParagraph = false;
       continue;
     }
     if (token.type === 'code-block') {
@@ -1748,12 +2242,14 @@ const parseBlockTokens = tokens => {
       flushParagraph();
       if (token.language) {
         nodes.push({
+          codeTokens: highlightCode(token.text, token.language),
           language: token.language,
           text: token.text,
           type: 'code-block'
         });
       } else {
         nodes.push({
+          codeTokens: highlightCode(token.text, undefined),
           text: token.text,
           type: 'code-block'
         });
@@ -1830,17 +2326,15 @@ const parseBlockTokens = tokens => {
         if (parentEntry) {
           const parentItem = getListItemAtPath(listItems, parentEntry.path);
           if (parentItem) {
-            const nextItem = {
-              children: parseInlineNodes(token.text),
-              type: 'list-item'
-            };
             const nextIndex = parentItem.nestedItems?.length || 0;
+            const nextItem = createListItem(token.text, nextIndex + 1);
             listItems = appendNestedItemAtPath(listItems, parentEntry.path, nextItem, 'ordered-list');
             const nextPath = [...parentEntry.path, nextIndex];
             orderedListPathStack = [...orderedListPathStack.filter(entry => entry.indentation < token.indentation), {
               indentation: token.indentation,
               path: nextPath
             }];
+            canContinueOrderedListItemParagraph = true;
             continue;
           }
         }
@@ -1850,27 +2344,23 @@ const parseBlockTokens = tokens => {
       }
       flushParagraph();
       listType = 'ordered-list';
-      const nextItem = {
-        children: parseInlineNodes(token.text),
-        type: 'list-item'
-      };
+      const nextIndex = listItems.length;
+      const nextItem = createListItem(token.text, nextIndex + 1);
       listItems.push(nextItem);
-      const nextIndex = listItems.length - 1;
       orderedListPathStack = [...orderedListPathStack.filter(entry => entry.indentation < token.indentation), {
         indentation: token.indentation,
         path: [nextIndex]
       }];
+      canContinueOrderedListItemParagraph = true;
       continue;
     }
     if (token.type === 'unordered-list-item-line') {
       if (listType === 'ordered-list' && listItems.length > 0 && token.indentation > 0 && orderedListPathStack.length > 0) {
         const parentEntry = orderedListPathStack.toReversed().find(entry => entry.indentation < token.indentation);
         if (parentEntry) {
-          const nextItem = {
-            children: parseInlineNodes(token.text),
-            type: 'list-item'
-          };
+          const nextItem = createListItem(token.text);
           listItems = appendNestedItemAtPath(listItems, parentEntry.path, nextItem, 'unordered-list');
+          canContinueOrderedListItemParagraph = false;
           continue;
         }
       }
@@ -1879,10 +2369,8 @@ const parseBlockTokens = tokens => {
       }
       flushParagraph();
       listType = 'unordered-list';
-      listItems.push({
-        children: parseInlineNodes(token.text),
-        type: 'list-item'
-      });
+      listItems.push(createListItem(token.text));
+      canContinueOrderedListItemParagraph = false;
       continue;
     }
     if (token.type === 'heading-line') {
@@ -1895,8 +2383,16 @@ const parseBlockTokens = tokens => {
       });
       continue;
     }
+    if (token.type === 'paragraph-line' && listType === 'ordered-list' && listItems.length > 0 && canContinueOrderedListItemParagraph) {
+      const currentPath = orderedListPathStack.at(-1)?.path;
+      if (currentPath) {
+        listItems = appendInlineChildrenAtPath(listItems, currentPath, parseInlineNodes(token.text));
+        continue;
+      }
+    }
     flushList();
     paragraphLines.push(token.text);
+    canContinueOrderedListItemParagraph = false;
   }
   flushList();
   flushParagraph();
@@ -1907,6 +2403,15 @@ const parseMessageContent = rawMessage => {
   return parseBlockTokens(scanBlockTokens(rawMessage));
 };
 
+const parseMessage = async rawMessage => {
+  const parsedContent = parseMessageContent(rawMessage);
+  const nextParsedContent = [];
+  for (const node of parsedContent) {
+    nextParsedContent.push(await parseMathNode(node));
+  }
+  return nextParsedContent;
+};
+
 const parseMessageContents = rawMessages => {
   return rawMessages.map(parseMessageContent);
 };
@@ -1914,16 +2419,29 @@ const parseMessageContents = rawMessages => {
 const commandMap = {
   'ChatMessageParsing.parseMessageContent': parseMessageContent,
   'ChatMessageParsing.parseMessageContents': parseMessageContents,
+  'ChatParser.parseMessage': parseMessage,
   'ChatParser.parseMessageContent': parseMessageContent,
   'ChatParser.parseMessageContents': parseMessageContents,
   'HandleMessagePort.handleMessagePort': handleMessagePort,
-  initialize: (_, port) => handleMessagePort(port)
+  initialize: initialize
+};
+
+const sendMessagePortToChatMathWorker = async port => {
+  await sendMessagePortToChatMathWorker$1(port, 0);
+};
+const initializeChatMathWorker = async () => {
+  const rpc = await create$2({
+    commandMap: {},
+    send: sendMessagePortToChatMathWorker
+  });
+  set(rpc);
 };
 
 const listen = async () => {
-  await create({
+  await create$1({
     commandMap: commandMap
   });
+  await initializeChatMathWorker();
 };
 
 const main = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/chat-message-parsing-worker",
-  "version": "1.6.0",
+  "version": "1.8.0",
   "description": "Chat Message Parsing Worker",
   "repository": {
     "type": "git",