@lvce-editor/shared-process 0.11.8 → 0.11.9

package/extensions/builtin.language-basics-perl/extension.json

@@ -6,7 +6,8 @@
     {
       "id": "perl",
       "extensions": [".pl", ".pm", ".pod", ".t", ".pL", ".psgi"],
-      "tokenize": "src/tokenizePerl.js"
+      "tokenize": "src/tokenizePerl.js",
+      "firstLine": "^#!.*\\bperl\\b"
     }
   ]
 }

package/extensions/builtin.language-basics-perl/src/tokenizePerl.js

@@ -5,6 +5,7 @@ export const State = {
   TopLevelContent: 1,
   InsideString: 2,
   InsideLineComment: 3,
+  InsideSingleQuoteString: 4,
 }
 
 export const StateMap = {
@@ -25,6 +26,9 @@ export const TokenType = {
   VariableName: 8,
   Comment: 885,
   Text: 9,
+  KeywordControl: 10,
+  KeywordReturn: 11,
+  Function: 12,
 }
 
 export const TokenMap = {
@@ -38,11 +42,14 @@ export const TokenMap = {
   [TokenType.VariableName]: 'VariableName',
   [TokenType.Comment]: 'Comment',
   [TokenType.Text]: 'Text',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
+  [TokenType.Function]: 'Function',
 }
 
 const RE_LINE_COMMENT = /^#.*/s
 const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
-const RE_WHITESPACE = /^ +/
+const RE_WHITESPACE = /^\s+/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
 const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
@@ -55,11 +62,13 @@ const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_QUOTE_DOUBLE = /^"/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_KEYWORD =
-  /^(?:case|continue|default|die|no|else|elsif|eval|exit|for|foreach|given|goto|if|last|next|redo|require|return|select|switch|unless|until|use|wait|when|while)\b/
-
-const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
-const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
+  /^(?:case|continue|default|die|no|else|elsif|eval|exit|for|foreach|given|goto|if|last|next|push|redo|require|return|select|shift|sub|switch|unless|until|use|wait|when|while)\b/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_VARIABLE_NAME = /^[$a-zA-Z\_@%]+/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\\\-\?]/
 const RE_NUMERIC = /^\d+/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_QUOTE_SINGLE = /^'/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -96,7 +105,31 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
+          switch (next[0]) {
+            case 'unless':
+            case 'foreach':
+            case 'if':
+            case 'while':
+            case 'next':
+            case 'else':
+            case 'last':
+            case 'elsif':
+              token = TokenType.KeywordControl
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'shift':
+            case 'push':
+            case 'eval':
+            case 'exit':
+            case 'die':
+              token = TokenType.Function
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
           state = State.TopLevelContent
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
@@ -110,6 +143,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
           state = State.InsideString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.TopLevelContent
@@ -132,6 +168,17 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         throw new Error('no')
     }

package/extensions/builtin.language-basics-python/src/tokenizePython.js

@@ -16,6 +16,8 @@ export const State = {
   InsideDoubleQuoteString: 11,
   InsideLineComment: 12,
   InsideSingleQuoteString: 13,
+  InsideTripleDoubleQuoteString: 14,
+  InsideTripleSingleQuoteString: 15,
 }
 
 /**
@@ -69,11 +71,13 @@ export const TokenMap = {
 
 const RE_WHITESPACE = /^\s+/
 const RE_DOUBLE_QUOTE = /^"/
+const RE_TRIPLE_DOUBLE_QUOTE = /^"{3}/
 const RE_SINGLE_QUOTE = /^'/
 const RE_TEXT = /^.+/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
-const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^'\\]+/
+const RE_STRING_TRIPLE_DOUBLE_QUOTE_CONTENT = /^.+(?=""")/
 const RE_LINE_COMMENT = /^#/
 const RE_KEYWORD =
   /^(?:__debug__|Ellipsis|yield|with|while|try|true|True|return|raise|pass|or|NotImplemented|not|nonlocal|None|lambda|is|in|import|if|global|from|for|finally|False|false|except|else|elif|del|def|continue|class|break|await|async|assert|as|and)\b/
@@ -82,6 +86,14 @@ const RE_NUMERIC =
   /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
 const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%]+/
 const RE_FUNCTION_CALL_NAME = /^[\w]+(?=\s*\()/
+const RE_DECORATOR = /^@[\w]+/
+const RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES = /.*(?=""")/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES = /.*(?=''')/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON = /.*/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_BACKSLASH = /^\\/
+const RE_TRIPLE_SINGLE_QUOTE = /^'{3}/
+const RE_NUMERIC_OCTAL = /^0(?:o|O)?[0-7][0-7_]*(n)?\b/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -130,6 +142,14 @@ export const tokenizeLine = (line, lineState) => {
             case 'finally':
             case 'while':
             case 'for':
+            case 'continue':
+            case 'not':
+            case 'except':
+            case 'with':
+            case 'as':
+            case 'raise':
+            case 'break':
+            case 'pass':
               token = TokenType.KeywordControl
               break
             default:
@@ -149,6 +169,15 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC_OCTAL))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleDoubleQuoteString
+        } else if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleSingleQuoteString
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.InsideDoubleQuoteString
@@ -158,6 +187,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.InsideLineComment
+        } else if ((next = part.match(RE_DECORATOR))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -173,6 +205,12 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
         } else {
           throw new Error('no')
         }
@@ -184,6 +222,49 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_SINGLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleDoubleQuoteString:
+        if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleSingleQuoteString:
+        if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          next
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
         } else {
           throw new Error('no')
         }
@@ -209,3 +290,5 @@ export const tokenizeLine = (line, lineState) => {
     tokens,
   }
 }
+
+tokenizeLine(`'''test'''`, initialLineState) //?

package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js

@@ -3,8 +3,12 @@
  */
 export const State = {
   TopLevelContent: 1,
-  InsideString: 2,
+  InsideDoubleQuoteString: 2,
   InsideLineComment: 3,
+  InsideSingleQuoteString: 4,
+  InsideBackTickString: 5,
+  InsideEof: 6,
+  AfterKeywordFunction: 7,
 }
 
 export const StateMap = {
@@ -25,6 +29,10 @@ export const TokenType = {
   VariableName: 8,
   Comment: 885,
   Text: 9,
+  KeywordControl: 10,
+  Function: 11,
+  LanguageConstant: 12,
+  KeywordReturn: 13,
 }
 
 export const TokenMap = {
@@ -38,11 +46,15 @@ export const TokenMap = {
   [TokenType.VariableName]: 'VariableName',
   [TokenType.Comment]: 'Comment',
   [TokenType.Text]: 'Text',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.Function]: 'Function',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
 }
 
 const RE_LINE_COMMENT = /^#.*/s
 const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
-const RE_WHITESPACE = /^ +/
+const RE_WHITESPACE = /^\s+/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
 const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
@@ -50,20 +62,31 @@ const RE_COLON = /^:/
 const RE_PROPERTY_VALUE = /^[^;\}]+/
 const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
+const RE_ANYTHING_SHORT = /^[^\s"']+/
 const RE_ANYTHING = /^.+/s
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_QUOTE_DOUBLE = /^"/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_QUOTE_SINGLE = /^'/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
 const RE_KEYWORD =
-  /^(?:alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|popd|printf|pushd|pwd|read|readonly|set|shift|shopt|source|suspend|test|times|trap|true|type|ulimit|umask|unalias|unset|wait)\b/
-
-const RE_VARIABLE_NAME = /^[a-zA-Z\_\/\-]+/
-const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
-const RE_NUMERIC = /^\d+/
+  /^(?:7za|7zr|alias|ar|awk|bg|bind|break|builtin|bunzip|bunzip2|caller|cargo|case|cat|catdoc|cd|chmod|chown|command|compgen|complete|continue|cp|curl|cygstart|dde-open|dirs|disown|do|done|done|dpkg|echo|elif|else|emulate|enable|enlightenment_open|esac|eval|eval_gettext|exec|exit|expr|false|fc|fg|fi|for|function|gdbus|getopts|gio|gnome-open|grep|grub-mount|gvfs-open|gzip|hash|head|help|history|identity|if|in|ip|jobs|kde-open|kfmclient|kill|less|let|lha|logout|ls|ls|lunzip|lzip|lzma|miniunz|miniunzip|mkdir|mktemp|mv|node|npm|open|pcmanfm|pdftotext|popd|prezip-bin|printf|ps|pushd|pwd|rar|read|readlink|readonly|return|rm|rpm|sed|set|shift|shopt|sleep|snap|sort|source|stty|suspend|tail|tar|tee|test|texi2dvi|then|times|touch|trap|true|tty|type|ulimit|umask|umask|unalias|unarj|unmkinitramfs|unrar|unset|unwrapdiff|unzip|unzoo|wait|which|while|xprop|xz|zoo|zstd)\b/
+const RE_VARIABLE_NAME = /^[a-zA-Z\_\/\-\$][a-zA-Z\_\/\-\$#\d\-]*/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\!\|\+\&\>\)]/
+const RE_NUMERIC = /^\d+(?=\s|$)/
+const RE_FUNCTION_NAME = /^\w+(?=\s*\()/
+const RE_EOF_START = /^<<\s*([\w\!]+)/
+const RE_EOF_CONTENT = /.*/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_BACKSLASH_AT_END = /^\\$/
 
 export const initialLineState = {
   state: State.TopLevelContent,
   tokens: [],
+  knownFunctionNames: new Set(),
+  stringEnd: '',
 }
 
 /**
@@ -88,6 +111,8 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let stringEnd = lineState.stringEnd
+  const knownFunctionNames = new Set(lineState.knownFunctionNames)
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -96,23 +121,200 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
           state = State.TopLevelContent
+          switch (next[0]) {
+            case 'break':
+            case 'case':
+            case 'continue':
+            case 'do':
+            case 'done':
+            case 'elif':
+            case 'else':
+            case 'esac':
+            case 'fi':
+            case 'for':
+            case 'if':
+            case 'in':
+            case 'then':
+            case 'while':
+              token = TokenType.KeywordControl
+              break
+            case '7za':
+            case '7za':
+            case '7zr':
+            case 'alias':
+            case 'ar':
+            case 'awk':
+            case 'bg':
+            case 'bind':
+            case 'builtin':
+            case 'bunzip':
+            case 'bunzip2':
+            case 'caller':
+            case 'cargo':
+            case 'cat':
+            case 'catdoc':
+            case 'cd':
+            case 'chmod':
+            case 'chown':
+            case 'command':
+            case 'compgen':
+            case 'complete':
+            case 'cp':
+            case 'curl':
+            case 'dirs':
+            case 'disown':
+            case 'dpkg':
+            case 'echo':
+            case 'emulate':
+            case 'enable':
+            case 'eval':
+            case 'eval_gettext':
+            case 'exec':
+            case 'exit':
+            case 'fc':
+            case 'fg':
+            case 'getopts':
+            case 'grep':
+            case 'grub-mount':
+            case 'gzip':
+            case 'hash':
+            case 'head':
+            case 'help':
+            case 'history':
+            case 'identity':
+            case 'jobs':
+            case 'kill':
+            case 'less':
+            case 'lha':
+            case 'logout':
+            case 'ls':
+            case 'lunzip':
+            case 'lzip':
+            case 'lzma':
+            case 'miniunz':
+            case 'miniunzip':
+            case 'mkdir':
+            case 'mktemp':
+            case 'mv':
+            case 'node':
+            case 'npm':
+            case 'pdftotext':
+            case 'popd':
+            case 'printf':
+            case 'ps':
+            case 'pushd':
+            case 'pwd':
+            case 'rar':
+            case 'read':
+            case 'rm':
+            case 'rpm':
+            case 'sed':
+            case 'set':
+            case 'shift':
+            case 'shopt':
+            case 'sleep':
+            case 'snap':
+            case 'source':
+            case 'stty':
+            case 'suspend':
+            case 'tar':
+            case 'tee':
+            case 'test':
+            case 'times':
+            case 'trap':
+            case 'type':
+            case 'ulimit':
+            case 'umask':
+            case 'umask':
+            case 'unalias':
+            case 'unarj':
+            case 'unmkinitramfs':
+            case 'unrar':
+            case 'unset':
+            case 'unwrapdiff':
+            case 'unzip':
+            case 'unzoo':
+            case 'wait':
+            case 'which':
+            case 'xz':
+            case 'zoo':
+            case 'zstd':
+            case 'expr':
+            case 'tty':
+            case 'sort':
+            case 'ls':
+            case 'readlink':
+            case 'gio':
+            case 'gvfs-open':
+            case 'gnome-open':
+            case 'xprop':
+            case 'cygstart':
+            case 'open':
+            case 'kde-open':
+            case 'kfmclient':
+            case 'dde-open':
+            case 'enlightenment_open':
+            case 'gdbus':
+            case 'pcmanfm':
+            case 'texi2dvi':
+            case 'tail':
+            case 'touch':
+            case 'prezip-bin':
+            case 'ip':
+              token = TokenType.Function
+              break
+            case 'true':
+            case 'false':
+              token = TokenType.LanguageConstant
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'function':
+              token = TokenType.Keyword
+              state = State.AfterKeywordFunction
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
+        } else if ((next = part.match(RE_EOF_START))) {
+          token = TokenType.Punctuation
+          state = State.InsideEof
+          stringEnd = next[1]
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_FUNCTION_NAME))) {
+          token = TokenType.Function
+          state = State.TopLevelContent
+          knownFunctionNames.add(next[0])
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
-          token = TokenType.VariableName
+          if (knownFunctionNames.has(next[0])) {
+            token = TokenType.Function
+          } else {
+            token = TokenType.VariableName
+          }
           state = State.TopLevelContent
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.InsideBackTickString
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING_SHORT))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -121,13 +323,70 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
-      case State.InsideString:
+      case State.InsideDoubleQuoteString:
         if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
         } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
           token = TokenType.String
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH_AT_END))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH_AT_END))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBackTickString:
+        if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideBackTickString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideEof:
+        if ((next = part.match(stringEnd))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EOF_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideEof
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterKeywordFunction:
+        if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.Function
+          state = State.TopLevelContent
+          knownFunctionNames.add(next[0])
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterKeywordFunction
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else {
           throw new Error('no')
         }
@@ -142,5 +401,7 @@ export const tokenizeLine = (line, lineState) => {
   return {
     state,
     tokens,
+    knownFunctionNames,
+    stringEnd,
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/shared-process",
-  "version": "0.11.8",
+  "version": "0.11.9",
   "description": "Utility package for @lvce-editor/server",
   "main": "index.js",
   "type": "module",
@@ -17,9 +17,9 @@
   },
   "dependencies": {
     "@babel/code-frame": "^7.18.6",
-    "@lvce-editor/extension-host": "0.11.8",
-    "@lvce-editor/extension-host-helper-process": "0.11.8",
-    "@lvce-editor/pty-host": "0.11.8",
+    "@lvce-editor/extension-host": "0.11.9",
+    "@lvce-editor/extension-host-helper-process": "0.11.9",
+    "@lvce-editor/pty-host": "0.11.9",
     "debug": "^4.3.4",
     "execa": "^6.1.0",
     "exit-hook": "^3.1.4",

package/src/parts/ErrorCodes/ErrorCodes.js

@@ -1,23 +1,13 @@
-export const ENOENT = 'ENOENT'
-
-export const EXDEV = 'EXDEV'
-
-export const EEXIST = 'EEXIST'
-
-export const ENOTDIR = 'ENOTDIR'
-
-export const E_MANIFEST_NOT_FOUND = 'E_MANIFEST_NOT_FOUND'
-
-export const EPIPE = 'EPIPE'
-
-export const ERR_IPC_CHANNEL_CLOSED = 'ERR_IPC_CHANNEL_CLOSED'
-
 export const E_COLOR_THEME_NOT_FOUND = 'E_COLOR_THEME_NOT_FOUND'
-
 export const E_ICON_THEME_NOT_FOUND = 'E_ICON_THEME_NOT_FOUND'
-
+export const E_MANIFEST_NOT_FOUND = 'E_MANIFEST_NOT_FOUND'
 export const ECONNRESET = 'ECONNRESET'
-
+export const EEXIST = 'EEXIST'
+export const ELOOP = 'ELOOP'
+export const ENOENT = 'ENOENT'
+export const ENOTDIR = 'ENOTDIR'
 export const EPERM = 'EPERM'
-
+export const EPIPE = 'EPIPE'
+export const ERR_IPC_CHANNEL_CLOSED = 'ERR_IPC_CHANNEL_CLOSED'
 export const ERR_MODULE_NOT_FOUND = 'ERR_MODULE_NOT_FOUND'
+export const EXDEV = 'EXDEV'