@lvce-editor/shared-process 0.12.0 → 0.13.0

package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js

@@ -72,7 +72,7 @@ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
 const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
 const RE_KEYWORD =
-  /^(?:7z|7za|7zr|alias|apk|apt-get|ar|asciidoctor|awk|basename|bash|bg|bind|blkid|break|brew|brotli|builtin|bunzip|bunzip2|caller|cargo|case|cat|catdoc|cd|chmod|chown|cmp|command|compgen|complete|continue|cp|cryptsetup|curl|curl|cut|cygstart|dde-open|defaults|diff|dirname|dirs|disown|do|docker|done|done|dpkg|du|echo|elif|else|emulate|enable|enlightenment_open|esac|eval|eval_gettext|exec|exit|expr|false|fc|fg|fi|find|for|free|function|fzf|g\+\+|gcc|gcloud|gdbus|getopts|gio|git|gnome-open|grep|grub-mount|gs|gulp|gvfs-open|gzip|hash|head|help|hexdump|history|identity|if|in|install|ip|jobs|kde-open|kfmclient|kill|killall|less|let|lha|ln|local|logout|ls|ls|lunzip|lzip|lzma|makepkg|miniunz|miniunzip|mkdir|mktemp|mount|mpicc|mv|node|npm|npx|open|pacman|pcmanfm|pdftotext|pkg|plutil|popd|prezip-bin|printf|println|ps|pushd|pwd|python3|rar|read|readlink|readonly|return|rm|rpm|rsync|sed|set|sha256sum|shift|shopt|sleep|snap|sort|source|stty|sudo|suspend|tail|tar|tee|test|texi2dvi|then|times|touch|trap|true|tty|tune2fs|type|ulimit|umask|umask|unalias|unarj|unmkinitramfs|unmount|unrar|unset|unwrapdiff|unzip|unzoo|update-alternatives|vim|wait|wc|wget|which|while|xargs|xprop|xz|yarn|yum|zip|zoo|zsh|zstd)\b/
+  /^(?:7z|7za|7zr|alias|anacron|apk|apt-get|ar|asciidoctor|awk|base64|basename|bash|bg|bind|blkid|break|brew|brotli|builtin|bunzip|bunzip2|caller|cargo|case|cat|catdoc|cd|chmod|chown|circleci|clear|cmp|command|compgen|complete|continue|cp|cryptsetup|curl|curl|cut|cygstart|dde-open|defaults|diff|dirname|dirs|disown|do|docker|done|done|dpkg-trigger|dpkg|du|echo|egrep|elif|else|emulate|enable|enlightenment_open|esac|eval|eval_gettext|exec|exit|expr|false|fc|fg|fi|find|for|free|function|fzf|g\+\+|gatsby|gcc|gcloud|gdbus|getopts|gio|git|gnome-open|grep|grub-mount|gs|gulp|gvfs-open|gzip|hash|head|help|hexdump|history|identity|if|ifconfig|in|install-plugin|install|ip|jobs|jq|kbd_mode|kde-open|kdialog|kfmclient|kill|killall|less|let|lha|ln|loadkeys|local|logout|ls|ls|lunzip|lzip|lzma|makepkg|miniunz|miniunzip|mkdir|mktemp|mount|mpicc|mv|node|npm|npx|open|pacman|pcmanfm|pdftotext|pkg|plutil|pnpm|popd|prezip-bin|printf|println|ps|pushd|pwd|python3|rar|read|readlink|readonly|return|rm|route|rpm|rsync|sed|set|setfont|sh|sha256sum|shift|shopt|sleep|snap|sort|source|sponge|start-stop-daemon|stty|sudo|suspend|tail|tar|tee|test|texi2dvi|then|times|touch|trap|true|tty|tune2fs|type|ulimit|umask|umask|unalias|unarj|unmkinitramfs|unmount|unrar|unset|unwrapdiff|unzip|unzoo|update-alternatives|vim|wait|wc|wget|which|while|wp|xargs|xprop|xz|yarn|yum|zenity|zip|zoo|zsh|zstd)\b/
 const RE_VARIABLE_NAME = /^[a-zA-Z\_\/\-\$][a-zA-Z\_\/\-\$#\d\-]*/
 const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\!\|\+\&\>\)]/
 const RE_NUMERIC = /^\d+(?=\s|$)/
@@ -144,11 +144,13 @@ export const tokenizeLine = (line, lineState) => {
             case '7za':
             case '7zr':
             case 'alias':
+            case 'anacron':
             case 'apk':
             case 'apt-get':
             case 'ar':
             case 'asciidoctor':
             case 'awk':
+            case 'base64':
             case 'basename':
             case 'bash':
             case 'bg':
@@ -166,6 +168,8 @@ export const tokenizeLine = (line, lineState) => {
             case 'cd':
             case 'chmod':
             case 'chown':
+            case 'circleci':
+            case 'clear':
             case 'cmp':
             case 'command':
             case 'compgen':
@@ -184,8 +188,10 @@ export const tokenizeLine = (line, lineState) => {
             case 'disown':
             case 'docker':
             case 'dpkg':
+            case 'dpkg-trigger':
             case 'du':
             case 'echo':
+            case 'egrep':
             case 'emulate':
             case 'enable':
             case 'enlightenment_open':
@@ -200,6 +206,7 @@ export const tokenizeLine = (line, lineState) => {
             case 'free':
             case 'fzf':
             case 'g++':
+            case 'gatsby':
             case 'gcc':
             case 'gcloud':
             case 'gdbus':
@@ -219,16 +226,21 @@ export const tokenizeLine = (line, lineState) => {
             case 'hexdump':
             case 'history':
             case 'identity':
+            case 'ifconfig':
             case 'install':
+            case 'install-plugin':
             case 'ip':
             case 'jobs':
+            case 'kbd_mode':
             case 'kde-open':
+            case 'kdialog':
             case 'kfmclient':
             case 'kill':
             case 'killall':
             case 'less':
             case 'lha':
             case 'ln':
+            case 'loadkeys':
             case 'logout':
             case 'ls':
             case 'ls':
@@ -252,6 +264,7 @@ export const tokenizeLine = (line, lineState) => {
             case 'pdftotext':
             case 'pkg':
             case 'plutil':
+            case 'pnpm':
             case 'popd':
             case 'prezip-bin':
             case 'printf':
@@ -264,10 +277,13 @@ export const tokenizeLine = (line, lineState) => {
             case 'read':
             case 'readlink':
             case 'rm':
+            case 'route':
             case 'rpm':
             case 'rsync':
             case 'sed':
             case 'set':
+            case 'setfont':
+            case 'sh':
             case 'sha256sum':
             case 'shift':
             case 'shopt':
@@ -275,6 +291,7 @@ export const tokenizeLine = (line, lineState) => {
             case 'snap':
             case 'sort':
             case 'source':
+            case 'start-stop-daemon':
             case 'stty':
             case 'sudo':
             case 'suspend':
@@ -307,15 +324,19 @@ export const tokenizeLine = (line, lineState) => {
             case 'wc':
             case 'wget':
             case 'which':
+            case 'wp':
             case 'xargs':
             case 'xprop':
             case 'xz':
             case 'yarn':
             case 'yum':
+            case 'zenity':
             case 'zip':
             case 'zoo':
             case 'zsh':
             case 'zstd':
+            case 'jq':
+            case 'sponge':
               token = TokenType.Function
               break
             case 'true':

package/extensions/builtin.language-basics-toml/src/tokenizeToml.js

@@ -9,6 +9,7 @@ export const State = {
   InsideSingleQuoteString: 6,
   InsideArray: 7,
   InsideObject: 8,
+  InsideTripleDoubleQuoteString: 9,
 }
 
 export const StateMap = {
@@ -58,7 +59,7 @@ const RE_LINE_COMMENT = /^#.*/s
 const RE_WHITESPACE = /^ +/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
-const RE_PROPERTY_NAME = /^[a-zA-Z\-\_\d]+\b(?=\s*=)/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-\_\d\.]+\b(?=\s*=)/
 const RE_COLON = /^:/
 const RE_PROPERTY_VALUE = /^[^;\}]+/
 const RE_SEMICOLON = /^;/
@@ -90,6 +91,9 @@ const RE_QUOTE_DOUBLE = /^"/
 const RE_QUOTE_SINGLE = /^'/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_TRIPLE_DOUBLE_QUOTE = /^"{3}/
+const RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES = /.*(?=""")/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON = /.*/s
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -168,6 +172,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_INF))) {
           token = TokenType.Numeric
           state = stack.pop() || State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleDoubleQuoteString
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.Punctuation
           state = State.InsideDoubleQuoteString
@@ -226,6 +233,13 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC_INTEGER))) {
           token = TokenType.Numeric
           state = State.InsideArray
+        } else if ((next = part.match(RE_SQUARE_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.InsideArray
+          stack.push(State.InsideArray)
+        } else if ((next = part.match(RE_LINE_COMMENT))) {
+          token = TokenType.Comment
+          state = State.InsideArray
         } else {
           part
           throw new Error('no')
@@ -256,6 +270,20 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideTripleDoubleQuoteString:
+        if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES))) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else if ((next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         throw new Error('no')
     }

package/extensions/builtin.language-basics-typescript/extension.json

@@ -5,7 +5,7 @@
   "languages": [
     {
       "id": "typescript",
-      "extensions": [".ts"],
+      "extensions": [".ts", ".mts", ".cts"],
       "tokenize": "src/tokenizeTypeScript.js"
     },
     {

package/extensions/builtin.language-basics-typescript/src/tokenizeTypeScript.js

@@ -34,6 +34,9 @@ const State = {
   InsideMethodParametersAfterVariableName: 31,
   AfterMethodParameters: 32,
   AfterPropertyDot: 33,
+  InsideBacktickString: 34,
+  AfterInterfaceName: 35,
+  AfterKeywordImport: 36,
   // AfterKeywordImport: 27,
 }
 
@@ -65,6 +68,7 @@ export const TokenType = {
   KeywordVoid: 19,
   Function: 20,
   FunctionName: 21,
+  KeywordOperator: 22,
 }
 
 export const TokenMap = {
@@ -92,6 +96,7 @@ export const TokenMap = {
   [TokenType.KeywordVoid]: 'KeywordVoid',
   [TokenType.Function]: 'Function',
   [TokenType.FunctionName]: 'Function',
+  [TokenType.KeywordOperator]: 'KeywordOperator',
 }
 
 export const initialLineState = {
@@ -106,14 +111,15 @@ const RE_LINE_COMMENT = /^\/\/[^\n]*/
 const RE_KEYWORD =
   /^(?:yield|with|while|void|var|typeof|type|true|try|throw|this|static|switch|super|readonly|return|public|protected|private|package|null|new|let|interface|instanceof|in|import|implements|if|function|for|finally|from|false|extends|export|enum|else|do|delete|default|debugger|declare|continue|const|constructor|class|catch|case|break|await|async|abstract)\b/
 const RE_WHITESPACE = /^\s+/
-const RE_VARIABLE_NAME = /^[\$a-zA-Z\_][\$a-zA-Z\_\d]*/
-const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>\+]/
+const RE_VARIABLE_NAME = /^[\#\$a-zA-Z\_][\$a-zA-Z\_\d]*/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)>\+\-]/
 const RE_QUOTE_SINGLE = /^'/
 const RE_QUOTE_DOUBLE = /^"/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^\\']+/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^\\"]+/
 const RE_NUMERIC = /^\d+/
 const RE_COLON = /^\:/
+const RE_COLON_OPTIONAL = /^\??\:/
 const RE_TYPE_PRIMITIVE = /^(?:string|boolean|number|bigint|symbol|void|any)\b/
 const RE_EQUAL = /^=/
 const RE_SEMICOLON = /^;/
@@ -161,9 +167,15 @@ const RE_NUMERIC_2 =
 const RE_NUMERIC_HEX = /0(?:x|X)[0-9a-fA-F][0-9a-fA-F_]*(n)?\b/
 const RE_NUMERIC_BINARY = /0(?:b|B)[01][01_]*(n)?\b/
 const RE_NUMERIC_OCTAL = /0(?:o|O)?[0-7][0-7_]*(n)?\b/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`\\]+/
+const RE_STRING_ESCAPE = /^\\./
+const RE_KEYWORD_TYPE = /^type\b/
+const RE_KEYWORD_IN = /^in\b/
+const RE_KEYWORD_OF = /^of\b/
+const RE_KEYWORD_EXTENDS = /^extends\b/
 
 export const hasArrayReturn = true
-
 /**
  *
  * @param {string} line
@@ -193,6 +205,9 @@ export const tokenizeLine = (line, lineState) => {
               state = State.TopLevelContent
               break
             case 'import':
+              token = TokenType.KeywordImport
+              state = State.AfterKeywordImport
+              break
             case 'export':
             case 'from':
               token = TokenType.KeywordImport
@@ -210,6 +225,7 @@ export const tokenizeLine = (line, lineState) => {
             case 'try':
             case 'catch':
             case 'finally':
+            case 'for':
               token = TokenType.KeywordControl
               state = State.TopLevelContent
               break
@@ -256,6 +272,13 @@ export const tokenizeLine = (line, lineState) => {
               token = TokenType.KeywordModifier
               state = State.TopLevelContent
               break
+            case 'in':
+            case 'of':
+            case 'typeof':
+            case 'instanceof':
+              token = TokenType.KeywordOperator
+              state = State.TopLevelContent
+              break
             default:
               token = TokenType.Keyword
               state = State.TopLevelContent
@@ -274,17 +297,15 @@ export const tokenizeLine = (line, lineState) => {
           } else if ((next = part.match(RE_LINE_COMMENT_START))) {
             token = TokenType.Comment
             state = State.InsideLineComment
+          } else if ((next = part.match(RE_PUNCTUATION))) {
+            token = TokenType.Punctuation
+            state = State.TopLevelContent
           } else {
-            next = part.match(RE_PUNCTUATION)
+            next = part.match(RE_SLASH)
             token = TokenType.Punctuation
             state = State.TopLevelContent
           }
-        }
-        //  else if ((next = part.match(RE_NUMERIC))) {
-        //   token = TokenType.Numeric
-        //   state = State.TopLevelContent
-        // }
-        else if ((next = part.match(RE_NUMERIC_2))) {
+        } else if ((next = part.match(RE_NUMERIC_2))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
         } else if ((next = part.match(RE_PUNCTUATION))) {
@@ -299,6 +320,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.Punctuation
           state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.InsideBacktickString
         } else if ((next = part.match(RE_OPERATOR))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
@@ -348,6 +372,12 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_KEYWORD_ENUM))) {
           token = TokenType.Keyword
           state = State.AfterKeywordEnum
+        } else if ((next = part.match(RE_KEYWORD_IN))) {
+          token = TokenType.KeywordOperator
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_KEYWORD_OF))) {
+          token = TokenType.KeywordOperator
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
           token = TokenType.VariableName
           state = State.AfterKeywordDeclaration
@@ -360,6 +390,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_DOT))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_SEMICOLON))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
         } else {
           line
           part
@@ -419,6 +452,9 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Punctuation
           state = State.InsideSingleQuoteString
           stack.push(State.BeforeType)
+        } else if ((next = part.match(RE_VERTICAL_LINE))) {
+          token = TokenType.Punctuation
+          state = State.BeforeType
         } else {
           part
           throw new Error('no')
@@ -530,6 +566,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_ANGLE_CLOSE))) {
           token = TokenType.Punctuation
           state = State.AfterType
+        } else if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.InsideTypeObject
         } else {
           // stack.push(State.AfterType)
           part
@@ -590,6 +629,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
           token = TokenType.VariableName
           state = State.InsideEnumAfterVariableName
+        } else if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop() || State.TopLevelContent
         } else {
           throw new Error('no')
         }
@@ -601,7 +643,14 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_EQUAL))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_COMMA))) {
+          token = TokenType.Punctuation
+          state = State.InsideEnum
+        } else if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop() || State.TopLevelContent
         } else {
+          part
           throw new Error('no')
         }
         break
@@ -712,7 +761,7 @@ export const tokenizeLine = (line, lineState) => {
           state = State.AfterKeywordInterface
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
           token = TokenType.Type
-          state = State.BeforeType
+          state = State.AfterInterfaceName
         } else {
           throw new Error('no')
         }
@@ -734,7 +783,7 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_CURLY_CLOSE))) {
           token = TokenType.Punctuation
           state = stack.pop() || State.TopLevelContent
-        } else if ((next = part.match(RE_COLON))) {
+        } else if ((next = part.match(RE_COLON_OPTIONAL))) {
           token = TokenType.Punctuation
           state = State.BeforeType
           stack.push(State.InsideTypeObject)
@@ -744,6 +793,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_SEMICOLON))) {
           token = TokenType.Punctuation
           state = State.InsideTypeObject
+        } else if ((next = part.match(RE_VERTICAL_LINE))) {
+          token = TokenType.Punctuation
+          state = State.BeforeType
         } else {
           part
           throw new Error('no')
@@ -817,6 +869,70 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideBacktickString:
+        if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideBacktickString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideBacktickString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterInterfaceName:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterInterfaceName
+        } else if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.InsideTypeObject
+        } else if ((next = part.match(RE_KEYWORD_EXTENDS))) {
+          token = TokenType.KeywordModifier
+          state = State.AfterInterfaceName
+        } else if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.Type
+          state = State.AfterInterfaceName
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
+      case State.AfterKeywordImport:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterKeywordImport
+        } else if ((next = part.match(RE_KEYWORD_TYPE))) {
+          token = TokenType.Keyword
+          state = State.AfterKeywordImport
+        } else if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STAR))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_ROUND_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_DOT))) {
+          token = TokenType.Punctuation
+          state = State.AfterPropertyDot
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         state
         throw new Error('no')

package/extensions/builtin.language-basics-xml/extension.json

@@ -69,9 +69,10 @@
         ".xpdl",
         ".xsd",
         ".xul",
-        ".tmtheme"
+        ".tmtheme",
+        ".mm"
       ],
-      "fileNames": [".project"],
+      "fileNames": [".project", "resources.grd"],
       "tokenize": "src/tokenizeXml.js"
     }
   ]

package/extensions/builtin.language-basics-xml/src/tokenizeXml.js

@@ -56,9 +56,10 @@ export const TokenMap = {
 const RE_ANGLE_BRACKET_CLOSE = /^>/
 const RE_ANGLE_BRACKET_ONLY = /^</
 const RE_ANGLE_BRACKET_OPEN = /^</
-const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
+const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?![\s!\?])/
+const RE_ANGLE_BRACKET_QUESTION_MARK = /^<\?/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-\:]+/
+const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-\:\_]+/
 const RE_BLOCK_COMMENT_CONTENT = /^.(?:.*?)(?=-->|$)/s
 const RE_BLOCK_COMMENT_END = /^-->/
 const RE_BLOCK_COMMENT_START = /^<!--/
@@ -74,6 +75,7 @@ const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
 const RE_PUNCTUATION = /^[<;'".,]/
 const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
 const RE_SELF_CLOSING = /^\/>/
+const RE_QUESTION_MARK_CLOSING = /^\?>/
 const RE_SINGLE_QUOTE = /^'/
 const RE_SLASH = /^\//
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
@@ -84,6 +86,9 @@ const RE_TEXT = /^[^<>\n]+/
 const RE_WHITESPACE = /^\s+/
 const RE_WORD = /^[^\s]+/
 const RE_QUESTION_MARK_TAG_NAME = /^\?\w+/
+const RE_BLOCK_COMMENT_CONTENT_1 = /^.+?(?=-->)/s
+const RE_BLOCK_COMMENT_CONTENT_2 = /^.+$/s
+const RE_SPECIAL_TAG = /^<!(?=\w)/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -123,9 +128,18 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.Text
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_SPECIAL_TAG))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterOpeningAngleBracket
+        } else if ((next = part.match(RE_ANGLE_BRACKET_QUESTION_MARK))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterOpeningAngleBracket
         } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -159,6 +173,9 @@ export const tokenizeLine = (line, lineState) => {
         if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUESTION_MARK_CLOSING))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_EXCLAMATION_MARK))) {
           token = TokenType.PunctuationTag
           state = State.InsideOpeningTag
@@ -206,6 +223,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_SELF_CLOSING))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUESTION_MARK_CLOSING))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
@@ -280,12 +300,15 @@ export const tokenizeLine = (line, lineState) => {
         }
         break
       case State.InsideBlockComment:
-        if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
+        if ((next = part.match(RE_BLOCK_COMMENT_CONTENT_1))) {
           token = TokenType.Comment
           state = State.InsideBlockComment
         } else if ((next = part.match(RE_BLOCK_COMMENT_END))) {
           token = TokenType.Comment
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_BLOCK_COMMENT_CONTENT_2))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
         } else {
           throw new Error('no')
         }

package/extensions/builtin.language-basics-yaml/src/tokenizeYaml.js

@@ -94,7 +94,7 @@ const RE_COMBINATOR = /^[\+\>\~]/
 const RE_LANGUAGE_CONSTANT = /^(?:true|false|null)(?!#)/
 const RE_COLON = /^:/
 const RE_DASH = /^\-/
-const RE_WORDS = /^[\w\s]*\w/
+const RE_WORDS = /^[\w\s\-\.]*\w/
 const RE_MULTI_LINE_STRING_START = /^(?:\||>\-|>)/
 const RE_KEY_PRE = /^\s*(\-\s*)?/
 const RE_SINGLE_QUOTE = /^'/
@@ -252,6 +252,12 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Comment
           state = State.InsideLineComment
           stack.push(State.AfterPipe)
+        } else if ((next = part.match(RE_DASH))) {
+          token = TokenType.Punctuation
+          state = State.AfterPipe
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.AfterPipe
         } else {
           throw new Error('no')
         }

package/extensions/builtin.theme-atom-one-dark/color-theme.json

@@ -188,6 +188,10 @@
     {
       "name": "MarkdownString",
       "foreground": "#ABB2BF"
+    },
+    {
+      "name": "KeywordModifier",
+      "foreground": "#C678DD"
     }
   ]
 }

package/extensions/builtin.theme-cobalt2/color-theme.json

@@ -171,6 +171,10 @@
     {
       "name": "Heading",
       "foreground": "#FFC600"
+    },
+    {
+      "name": "KeywordNew",
+      "foreground": "#FF9D00"
     }
   ]
 }

package/extensions/builtin.theme-gruvbox/color-theme.json

@@ -208,6 +208,10 @@
     {
       "name": "MarkdownString",
       "foreground": "#A9B665"
+    },
+    {
+      "name": "KeywordModifier",
+      "foreground": "#E78A4E"
     }
   ]
 }