@lvce-editor/shared-process 0.11.7 → 0.11.9

package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js

@@ -3,8 +3,12 @@
  */
 export const State = {
   TopLevelContent: 1,
-  InsideString: 2,
+  InsideDoubleQuoteString: 2,
   InsideLineComment: 3,
+  InsideSingleQuoteString: 4,
+  InsideBackTickString: 5,
+  InsideEof: 6,
+  AfterKeywordFunction: 7,
 }
 
 export const StateMap = {
@@ -25,6 +29,10 @@ export const TokenType = {
   VariableName: 8,
   Comment: 885,
   Text: 9,
+  KeywordControl: 10,
+  Function: 11,
+  LanguageConstant: 12,
+  KeywordReturn: 13,
 }
 
 export const TokenMap = {
@@ -38,11 +46,15 @@ export const TokenMap = {
   [TokenType.VariableName]: 'VariableName',
   [TokenType.Comment]: 'Comment',
   [TokenType.Text]: 'Text',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.Function]: 'Function',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
 }
 
 const RE_LINE_COMMENT = /^#.*/s
 const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
-const RE_WHITESPACE = /^ +/
+const RE_WHITESPACE = /^\s+/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
 const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
@@ -50,20 +62,31 @@ const RE_COLON = /^:/
 const RE_PROPERTY_VALUE = /^[^;\}]+/
 const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
+const RE_ANYTHING_SHORT = /^[^\s"']+/
 const RE_ANYTHING = /^.+/s
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_QUOTE_DOUBLE = /^"/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_QUOTE_SINGLE = /^'/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
 const RE_KEYWORD =
-  /^(?:alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|popd|printf|pushd|pwd|read|readonly|set|shift|shopt|source|suspend|test|times|trap|true|type|ulimit|umask|unalias|unset|wait)\b/
-
-const RE_VARIABLE_NAME = /^[a-zA-Z\_\/\-]+/
-const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
-const RE_NUMERIC = /^\d+/
+  /^(?:7za|7zr|alias|ar|awk|bg|bind|break|builtin|bunzip|bunzip2|caller|cargo|case|cat|catdoc|cd|chmod|chown|command|compgen|complete|continue|cp|curl|cygstart|dde-open|dirs|disown|do|done|done|dpkg|echo|elif|else|emulate|enable|enlightenment_open|esac|eval|eval_gettext|exec|exit|expr|false|fc|fg|fi|for|function|gdbus|getopts|gio|gnome-open|grep|grub-mount|gvfs-open|gzip|hash|head|help|history|identity|if|in|ip|jobs|kde-open|kfmclient|kill|less|let|lha|logout|ls|ls|lunzip|lzip|lzma|miniunz|miniunzip|mkdir|mktemp|mv|node|npm|open|pcmanfm|pdftotext|popd|prezip-bin|printf|ps|pushd|pwd|rar|read|readlink|readonly|return|rm|rpm|sed|set|shift|shopt|sleep|snap|sort|source|stty|suspend|tail|tar|tee|test|texi2dvi|then|times|touch|trap|true|tty|type|ulimit|umask|umask|unalias|unarj|unmkinitramfs|unrar|unset|unwrapdiff|unzip|unzoo|wait|which|while|xprop|xz|zoo|zstd)\b/
+const RE_VARIABLE_NAME = /^[a-zA-Z\_\/\-\$][a-zA-Z\_\/\-\$#\d\-]*/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\!\|\+\&\>\)]/
+const RE_NUMERIC = /^\d+(?=\s|$)/
+const RE_FUNCTION_NAME = /^\w+(?=\s*\()/
+const RE_EOF_START = /^<<\s*([\w\!]+)/
+const RE_EOF_CONTENT = /.*/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_BACKSLASH_AT_END = /^\\$/
 
 export const initialLineState = {
   state: State.TopLevelContent,
   tokens: [],
+  knownFunctionNames: new Set(),
+  stringEnd: '',
 }
 
 /**
@@ -88,6 +111,8 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let stringEnd = lineState.stringEnd
+  const knownFunctionNames = new Set(lineState.knownFunctionNames)
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -96,23 +121,200 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
           state = State.TopLevelContent
+          switch (next[0]) {
+            case 'break':
+            case 'case':
+            case 'continue':
+            case 'do':
+            case 'done':
+            case 'elif':
+            case 'else':
+            case 'esac':
+            case 'fi':
+            case 'for':
+            case 'if':
+            case 'in':
+            case 'then':
+            case 'while':
+              token = TokenType.KeywordControl
+              break
+            case '7za':
+            case '7za':
+            case '7zr':
+            case 'alias':
+            case 'ar':
+            case 'awk':
+            case 'bg':
+            case 'bind':
+            case 'builtin':
+            case 'bunzip':
+            case 'bunzip2':
+            case 'caller':
+            case 'cargo':
+            case 'cat':
+            case 'catdoc':
+            case 'cd':
+            case 'chmod':
+            case 'chown':
+            case 'command':
+            case 'compgen':
+            case 'complete':
+            case 'cp':
+            case 'curl':
+            case 'dirs':
+            case 'disown':
+            case 'dpkg':
+            case 'echo':
+            case 'emulate':
+            case 'enable':
+            case 'eval':
+            case 'eval_gettext':
+            case 'exec':
+            case 'exit':
+            case 'fc':
+            case 'fg':
+            case 'getopts':
+            case 'grep':
+            case 'grub-mount':
+            case 'gzip':
+            case 'hash':
+            case 'head':
+            case 'help':
+            case 'history':
+            case 'identity':
+            case 'jobs':
+            case 'kill':
+            case 'less':
+            case 'lha':
+            case 'logout':
+            case 'ls':
+            case 'lunzip':
+            case 'lzip':
+            case 'lzma':
+            case 'miniunz':
+            case 'miniunzip':
+            case 'mkdir':
+            case 'mktemp':
+            case 'mv':
+            case 'node':
+            case 'npm':
+            case 'pdftotext':
+            case 'popd':
+            case 'printf':
+            case 'ps':
+            case 'pushd':
+            case 'pwd':
+            case 'rar':
+            case 'read':
+            case 'rm':
+            case 'rpm':
+            case 'sed':
+            case 'set':
+            case 'shift':
+            case 'shopt':
+            case 'sleep':
+            case 'snap':
+            case 'source':
+            case 'stty':
+            case 'suspend':
+            case 'tar':
+            case 'tee':
+            case 'test':
+            case 'times':
+            case 'trap':
+            case 'type':
+            case 'ulimit':
+            case 'umask':
+            case 'umask':
+            case 'unalias':
+            case 'unarj':
+            case 'unmkinitramfs':
+            case 'unrar':
+            case 'unset':
+            case 'unwrapdiff':
+            case 'unzip':
+            case 'unzoo':
+            case 'wait':
+            case 'which':
+            case 'xz':
+            case 'zoo':
+            case 'zstd':
+            case 'expr':
+            case 'tty':
+            case 'sort':
+            case 'ls':
+            case 'readlink':
+            case 'gio':
+            case 'gvfs-open':
+            case 'gnome-open':
+            case 'xprop':
+            case 'cygstart':
+            case 'open':
+            case 'kde-open':
+            case 'kfmclient':
+            case 'dde-open':
+            case 'enlightenment_open':
+            case 'gdbus':
+            case 'pcmanfm':
+            case 'texi2dvi':
+            case 'tail':
+            case 'touch':
+            case 'prezip-bin':
+            case 'ip':
+              token = TokenType.Function
+              break
+            case 'true':
+            case 'false':
+              token = TokenType.LanguageConstant
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'function':
+              token = TokenType.Keyword
+              state = State.AfterKeywordFunction
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
+        } else if ((next = part.match(RE_EOF_START))) {
+          token = TokenType.Punctuation
+          state = State.InsideEof
+          stringEnd = next[1]
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_FUNCTION_NAME))) {
+          token = TokenType.Function
+          state = State.TopLevelContent
+          knownFunctionNames.add(next[0])
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
-          token = TokenType.VariableName
+          if (knownFunctionNames.has(next[0])) {
+            token = TokenType.Function
+          } else {
+            token = TokenType.VariableName
+          }
           state = State.TopLevelContent
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.InsideBackTickString
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING_SHORT))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -121,13 +323,70 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
-      case State.InsideString:
+      case State.InsideDoubleQuoteString:
         if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
         } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
           token = TokenType.String
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH_AT_END))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH_AT_END))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBackTickString:
+        if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideBackTickString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideEof:
+        if ((next = part.match(stringEnd))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EOF_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideEof
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterKeywordFunction:
+        if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.Function
+          state = State.TopLevelContent
+          knownFunctionNames.add(next[0])
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterKeywordFunction
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else {
           throw new Error('no')
         }
@@ -142,5 +401,7 @@ export const tokenizeLine = (line, lineState) => {
   return {
     state,
     tokens,
+    knownFunctionNames,
+    stringEnd,
   }
 }