@lvce-editor/chat-message-parsing-worker 1.7.0 → 1.8.0

package/dist/chatMessageParsingWorkerMain.js

@@ -1018,6 +1018,404 @@ const initialize = async (_, port) => {
   await handleMessagePort(port);
 };
 
+const createMockRpc = ({
+  commandMap
+}) => {
+  const invocations = [];
+  const invoke = (method, ...params) => {
+    invocations.push([method, ...params]);
+    const command = commandMap[method];
+    if (!command) {
+      throw new Error(`command ${method} not found`);
+    }
+    return command(...params);
+  };
+  const mockRpc = {
+    invocations,
+    invoke,
+    invokeAndTransfer: invoke
+  };
+  return mockRpc;
+};
+
+const rpcs = Object.create(null);
+const set$1 = (id, rpc) => {
+  rpcs[id] = rpc;
+};
+const get = id => {
+  return rpcs[id];
+};
+const remove = id => {
+  delete rpcs[id];
+};
+
+/* eslint-disable  @typescript-eslint/explicit-function-return-type */
+const create = rpcId => {
+  return {
+    async dispose() {
+      const rpc = get(rpcId);
+      await rpc.dispose();
+    },
+    // @ts-ignore
+    invoke(method, ...params) {
+      const rpc = get(rpcId);
+      // @ts-ignore
+      return rpc.invoke(method, ...params);
+    },
+    // @ts-ignore
+    invokeAndTransfer(method, ...params) {
+      const rpc = get(rpcId);
+      // @ts-ignore
+      return rpc.invokeAndTransfer(method, ...params);
+    },
+    registerMockRpc(commandMap) {
+      const mockRpc = createMockRpc({
+        commandMap
+      });
+      set$1(rpcId, mockRpc);
+      // @ts-ignore
+      mockRpc[Symbol.dispose] = () => {
+        remove(rpcId);
+      };
+      // @ts-ignore
+      return mockRpc;
+    },
+    set(rpc) {
+      set$1(rpcId, rpc);
+    }
+  };
+};
+
+const {
+  invoke,
+  set
+} = create(6007);
+const getMathBlockDom = async node => {
+  return invoke('ChatMath.getMathBlockDom', node);
+};
+const getMathInlineDom = async node => {
+  return invoke('ChatMath.getMathInlineDom', node);
+};
+
+const RendererWorker = 1;
+
+const {
+  invokeAndTransfer} = create(RendererWorker);
+const sendMessagePortToChatMathWorker$1 = async (port, rpcId) => {
+  const command = 'HandleMessagePort.handleMessagePort';
+  await invokeAndTransfer('SendMessagePortToExtensionHostWorker.sendMessagePortToChatMathWorker', port, command, rpcId);
+};
+
+const parseMathInline = async children => {
+  const nextChildren = [];
+  for (const child of children) {
+    if (child.type === 'math-inline') {
+      const dom = await getMathInlineDom(child);
+      nextChildren.push({
+        dom,
+        type: 'math-inline-dom'
+      });
+      continue;
+    }
+    if (child.type === 'bold') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    if (child.type === 'italic') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    if (child.type === 'strikethrough') {
+      nextChildren.push({
+        ...child,
+        children: await parseMathInline(child.children)
+      });
+      continue;
+    }
+    nextChildren.push(child);
+  }
+  return nextChildren;
+};
+const parseMathListItem = async item => {
+  const children = await parseMathInline(item.children);
+  if (!item.nestedItems) {
+    return {
+      ...item,
+      children
+    };
+  }
+  const nestedItems = [];
+  for (const nestedItem of item.nestedItems) {
+    nestedItems.push(await parseMathListItem(nestedItem));
+  }
+  return {
+    ...item,
+    children,
+    nestedItems
+  };
+};
+const parseMathTableCell = async cell => {
+  return {
+    ...cell,
+    children: await parseMathInline(cell.children)
+  };
+};
+const parseMathNode = async node => {
+  if (node.type === 'math-block') {
+    const dom = await getMathBlockDom(node);
+    return {
+      dom,
+      type: 'math-block-dom'
+    };
+  }
+  if (node.type === 'text' || node.type === 'heading') {
+    return {
+      ...node,
+      children: await parseMathInline(node.children)
+    };
+  }
+  if (node.type === 'blockquote') {
+    const children = [];
+    for (const child of node.children) {
+      children.push(await parseMathNode(child));
+    }
+    return {
+      ...node,
+      children
+    };
+  }
+  if (node.type === 'ordered-list' || node.type === 'unordered-list') {
+    const items = [];
+    for (const item of node.items) {
+      items.push(await parseMathListItem(item));
+    }
+    return {
+      ...node,
+      items
+    };
+  }
+  if (node.type === 'table') {
+    const headers = [];
+    for (const header of node.headers) {
+      headers.push(await parseMathTableCell(header));
+    }
+    const rows = [];
+    for (const row of node.rows) {
+      const cells = [];
+      for (const cell of row.cells) {
+        cells.push(await parseMathTableCell(cell));
+      }
+      rows.push({
+        ...row,
+        cells
+      });
+    }
+    return {
+      ...node,
+      headers,
+      rows
+    };
+  }
+  return node;
+};
+
+const TokenAttribute = 'TokenAttribute';
+const TokenComment = 'TokenComment';
+const TokenKeyword = 'TokenKeyword';
+const TokenNumber = 'TokenNumber';
+const TokenProperty = 'TokenProperty';
+const TokenString = 'TokenString';
+const TokenTag = 'TokenTag';
+const TokenValue = 'TokenValue';
+
+const jsRules = [{
+  className: TokenComment,
+  regex: /\/\/[^\n]*/
+}, {
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenString,
+  regex: /`[^`\\]*(?:\\.[^`\\]*)*`/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:const|let|var|function|return|if|else|for|while|class|new|typeof|instanceof|import|export|from|default|async|await|try|catch|finally|throw|this|true|false|null|undefined)\b/
+}];
+const tsRules = [{
+  className: TokenComment,
+  regex: /\/\/[^\n]*/
+}, {
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenString,
+  regex: /`[^`\\]*(?:\\.[^`\\]*)*`/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:abstract|any|as|asserts|async|await|boolean|class|const|constructor|declare|default|enum|export|extends|false|finally|for|from|function|get|if|implements|import|in|infer|instanceof|interface|is|keyof|let|module|namespace|never|new|null|number|object|override|private|protected|public|readonly|return|satisfies|set|static|string|super|switch|symbol|this|throw|true|try|type|typeof|undefined|unknown|var|void|while)\b/
+}];
+const htmlRules = [{
+  className: TokenComment,
+  regex: /<!--[\s\S]*?-->/
+}, {
+  className: TokenTag,
+  regex: /<\/[a-zA-Z][a-zA-Z0-9-]*\s*>/
+}, {
+  className: TokenTag,
+  regex: /<[a-zA-Z][a-zA-Z0-9-]*/
+}, {
+  className: TokenTag,
+  regex: /\/?>/
+}, {
+  className: TokenString,
+  regex: /"[^"]*"/
+}, {
+  className: TokenString,
+  regex: /'[^']*'/
+}, {
+  className: TokenAttribute,
+  regex: /[a-zA-Z-]+(?=\s*=)/
+}];
+const cssRules = [{
+  className: TokenComment,
+  regex: /\/\*[\s\S]*?\*\//
+}, {
+  className: TokenString,
+  regex: /"[^"]*"/
+}, {
+  className: TokenString,
+  regex: /'[^']*'/
+}, {
+  className: TokenValue,
+  regex: /#[0-9a-fA-F]{3,8}/
+}, {
+  className: TokenValue,
+  regex: /\b\d+(?:\.\d+)?(?:px|em|rem|%|vh|vw|pt|s|ms|fr)?\b/
+}, {
+  className: TokenProperty,
+  regex: /[a-zA-Z-]+(?=\s*:)/
+}];
+const pythonRules = [{
+  className: TokenComment,
+  regex: /#[^\n]*/
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenString,
+  regex: /'[^'\\]*(?:\\.[^'\\]*)*'/
+}, {
+  className: TokenNumber,
+  regex: /\b\d+\.?\d*\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:def|class|return|if|elif|else|for|while|in|import|from|as|with|try|except|finally|raise|lambda|yield|pass|break|continue|True|False|None|and|or|not|is)\b/
+}];
+const jsonRules = [{
+  className: TokenProperty,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"(?=\s*:)/
+}, {
+  className: TokenString,
+  regex: /"[^"\\]*(?:\\.[^"\\]*)*"/
+}, {
+  className: TokenNumber,
+  regex: /-?\b\d+\.?\d*(?:[eE][+-]?\d+)?\b/
+}, {
+  className: TokenKeyword,
+  regex: /\b(?:true|false|null)\b/
+}];
+const tokenize = (code, rules) => {
+  const tokens = [];
+  let pos = 0;
+  while (pos < code.length) {
+    let matched = false;
+    for (const rule of rules) {
+      const match = rule.regex.exec(code.slice(pos));
+      if (match && match.index === 0) {
+        tokens.push({
+          className: rule.className,
+          text: match[0]
+        });
+        pos += match[0].length;
+        matched = true;
+        break;
+      }
+    }
+    if (!matched) {
+      const last = tokens.at(-1);
+      if (last && last.className === '') {
+        tokens.pop();
+        tokens.push({
+          className: '',
+          text: last.text + code[pos]
+        });
+      } else {
+        tokens.push({
+          className: '',
+          text: code[pos]
+        });
+      }
+      pos++;
+    }
+  }
+  return tokens;
+};
+const highlightCode = (code, language) => {
+  if (!language) {
+    return [{
+      className: '',
+      text: code
+    }];
+  }
+  const normalized = language.toLowerCase();
+  if (normalized === 'html') {
+    return tokenize(code, htmlRules);
+  }
+  if (normalized === 'css') {
+    return tokenize(code, cssRules);
+  }
+  if (normalized === 'js' || normalized === 'javascript') {
+    return tokenize(code, jsRules);
+  }
+  if (normalized === 'ts' || normalized === 'typescript') {
+    return tokenize(code, tsRules);
+  }
+  if (normalized === 'py' || normalized === 'python') {
+    return tokenize(code, pythonRules);
+  }
+  if (normalized === 'json') {
+    return tokenize(code, jsonRules);
+  }
+  return [{
+    className: '',
+    text: code
+  }];
+};
+
 const windowsAbsolutePathRegex = /^[a-zA-Z]:[\\/]/;
 const pathSeparatorRegex$1 = /[\\/]/;
 const isPathTraversalAttempt = path => {
@@ -1738,8 +2136,6 @@ const getEmptyTextNode = () => {
     type: 'text'
   }];
 };
-
-// eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
 const parseBlockTokens = tokens => {
   if (tokens.length === 0) {
     return getEmptyTextNode();
@@ -1749,6 +2145,16 @@ const parseBlockTokens = tokens => {
   let listItems = [];
   let listType = '';
   let orderedListPathStack = [];
+  let canContinueOrderedListItemParagraph = false;
+  const createListItem = (text, index) => {
+    return {
+      children: parseInlineNodes(text),
+      ...(index === undefined ? {} : {
+        index
+      }),
+      type: 'list-item'
+    };
+  };
   const getListItemAtPath = (items, path) => {
     let currentItems = items;
     let currentItem;
@@ -1778,6 +2184,29 @@ const parseBlockTokens = tokens => {
     };
     return [...items.slice(0, index), nextItem, ...items.slice(index + 1)];
   };
+  const appendInlineChildrenAtPath = (items, path, children) => {
+    if (path.length === 0) {
+      return [...items];
+    }
+    const [index, ...rest] = path;
+    const current = items[index];
+    if (!current) {
+      return [...items];
+    }
+    const lineBreakNode = {
+      text: '\n',
+      type: 'text'
+    };
+    const nextChildren = [...current.children, lineBreakNode, ...children];
+    const nextItem = rest.length > 0 ? {
+      ...current,
+      nestedItems: appendInlineChildrenAtPath(current.nestedItems || [], rest, children)
+    } : {
+      ...current,
+      children: nextChildren
+    };
+    return [...items.slice(0, index), nextItem, ...items.slice(index + 1)];
+  };
   const flushParagraph = () => {
     if (paragraphLines.length === 0) {
       return;
@@ -1799,11 +2228,13 @@ const parseBlockTokens = tokens => {
     listItems = [];
     listType = '';
     orderedListPathStack = [];
+    canContinueOrderedListItemParagraph = false;
   };
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
     if (token.type === 'blank-line') {
       flushParagraph();
+      canContinueOrderedListItemParagraph = false;
       continue;
     }
     if (token.type === 'code-block') {
@@ -1811,12 +2242,14 @@ const parseBlockTokens = tokens => {
       flushParagraph();
       if (token.language) {
         nodes.push({
+          codeTokens: highlightCode(token.text, token.language),
           language: token.language,
           text: token.text,
           type: 'code-block'
         });
       } else {
         nodes.push({
+          codeTokens: highlightCode(token.text, undefined),
           text: token.text,
           type: 'code-block'
         });
@@ -1893,17 +2326,15 @@ const parseBlockTokens = tokens => {
         if (parentEntry) {
           const parentItem = getListItemAtPath(listItems, parentEntry.path);
           if (parentItem) {
-            const nextItem = {
-              children: parseInlineNodes(token.text),
-              type: 'list-item'
-            };
             const nextIndex = parentItem.nestedItems?.length || 0;
+            const nextItem = createListItem(token.text, nextIndex + 1);
             listItems = appendNestedItemAtPath(listItems, parentEntry.path, nextItem, 'ordered-list');
             const nextPath = [...parentEntry.path, nextIndex];
             orderedListPathStack = [...orderedListPathStack.filter(entry => entry.indentation < token.indentation), {
               indentation: token.indentation,
               path: nextPath
             }];
+            canContinueOrderedListItemParagraph = true;
             continue;
           }
         }
@@ -1913,27 +2344,23 @@ const parseBlockTokens = tokens => {
       }
       flushParagraph();
       listType = 'ordered-list';
-      const nextItem = {
-        children: parseInlineNodes(token.text),
-        type: 'list-item'
-      };
+      const nextIndex = listItems.length;
+      const nextItem = createListItem(token.text, nextIndex + 1);
       listItems.push(nextItem);
-      const nextIndex = listItems.length - 1;
       orderedListPathStack = [...orderedListPathStack.filter(entry => entry.indentation < token.indentation), {
         indentation: token.indentation,
         path: [nextIndex]
       }];
+      canContinueOrderedListItemParagraph = true;
       continue;
     }
     if (token.type === 'unordered-list-item-line') {
       if (listType === 'ordered-list' && listItems.length > 0 && token.indentation > 0 && orderedListPathStack.length > 0) {
         const parentEntry = orderedListPathStack.toReversed().find(entry => entry.indentation < token.indentation);
         if (parentEntry) {
-          const nextItem = {
-            children: parseInlineNodes(token.text),
-            type: 'list-item'
-          };
+          const nextItem = createListItem(token.text);
           listItems = appendNestedItemAtPath(listItems, parentEntry.path, nextItem, 'unordered-list');
+          canContinueOrderedListItemParagraph = false;
           continue;
         }
       }
@@ -1942,10 +2369,8 @@ const parseBlockTokens = tokens => {
       }
       flushParagraph();
       listType = 'unordered-list';
-      listItems.push({
-        children: parseInlineNodes(token.text),
-        type: 'list-item'
-      });
+      listItems.push(createListItem(token.text));
+      canContinueOrderedListItemParagraph = false;
       continue;
     }
     if (token.type === 'heading-line') {
@@ -1958,8 +2383,16 @@ const parseBlockTokens = tokens => {
       });
       continue;
     }
+    if (token.type === 'paragraph-line' && listType === 'ordered-list' && listItems.length > 0 && canContinueOrderedListItemParagraph) {
+      const currentPath = orderedListPathStack.at(-1)?.path;
+      if (currentPath) {
+        listItems = appendInlineChildrenAtPath(listItems, currentPath, parseInlineNodes(token.text));
+        continue;
+      }
+    }
     flushList();
     paragraphLines.push(token.text);
+    canContinueOrderedListItemParagraph = false;
   }
   flushList();
   flushParagraph();
@@ -1970,6 +2403,15 @@ const parseMessageContent = rawMessage => {
   return parseBlockTokens(scanBlockTokens(rawMessage));
 };
 
+const parseMessage = async rawMessage => {
+  const parsedContent = parseMessageContent(rawMessage);
+  const nextParsedContent = [];
+  for (const node of parsedContent) {
+    nextParsedContent.push(await parseMathNode(node));
+  }
+  return nextParsedContent;
+};
+
 const parseMessageContents = rawMessages => {
   return rawMessages.map(parseMessageContent);
 };
@@ -1977,93 +2419,13 @@ const parseMessageContents = rawMessages => {
 const commandMap = {
   'ChatMessageParsing.parseMessageContent': parseMessageContent,
   'ChatMessageParsing.parseMessageContents': parseMessageContents,
+  'ChatParser.parseMessage': parseMessage,
   'ChatParser.parseMessageContent': parseMessageContent,
   'ChatParser.parseMessageContents': parseMessageContents,
   'HandleMessagePort.handleMessagePort': handleMessagePort,
   initialize: initialize
 };
 
-const createMockRpc = ({
-  commandMap
-}) => {
-  const invocations = [];
-  const invoke = (method, ...params) => {
-    invocations.push([method, ...params]);
-    const command = commandMap[method];
-    if (!command) {
-      throw new Error(`command ${method} not found`);
-    }
-    return command(...params);
-  };
-  const mockRpc = {
-    invocations,
-    invoke,
-    invokeAndTransfer: invoke
-  };
-  return mockRpc;
-};
-
-const rpcs = Object.create(null);
-const set$1 = (id, rpc) => {
-  rpcs[id] = rpc;
-};
-const get = id => {
-  return rpcs[id];
-};
-const remove = id => {
-  delete rpcs[id];
-};
-
-/* eslint-disable  @typescript-eslint/explicit-function-return-type */
-const create = rpcId => {
-  return {
-    async dispose() {
-      const rpc = get(rpcId);
-      await rpc.dispose();
-    },
-    // @ts-ignore
-    invoke(method, ...params) {
-      const rpc = get(rpcId);
-      // @ts-ignore
-      return rpc.invoke(method, ...params);
-    },
-    // @ts-ignore
-    invokeAndTransfer(method, ...params) {
-      const rpc = get(rpcId);
-      // @ts-ignore
-      return rpc.invokeAndTransfer(method, ...params);
-    },
-    registerMockRpc(commandMap) {
-      const mockRpc = createMockRpc({
-        commandMap
-      });
-      set$1(rpcId, mockRpc);
-      // @ts-ignore
-      mockRpc[Symbol.dispose] = () => {
-        remove(rpcId);
-      };
-      // @ts-ignore
-      return mockRpc;
-    },
-    set(rpc) {
-      set$1(rpcId, rpc);
-    }
-  };
-};
-
-const {
-  set
-} = create(6007);
-
-const RendererWorker = 1;
-
-const {
-  invokeAndTransfer} = create(RendererWorker);
-const sendMessagePortToChatMathWorker$1 = async (port, rpcId) => {
-  const command = 'HandleMessagePort.handleMessagePort';
-  await invokeAndTransfer('SendMessagePortToExtensionHostWorker.sendMessagePortToChatMathWorker', port, command, rpcId);
-};
-
 const sendMessagePortToChatMathWorker = async port => {
   await sendMessagePortToChatMathWorker$1(port, 0);
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/chat-message-parsing-worker",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "Chat Message Parsing Worker",
   "repository": {
     "type": "git",