@lvce-editor/shared-process 0.0.37 → 0.0.38

package/extensions/builtin.language-basics-c/README.md

@@ -0,0 +1,14 @@
+# Language Basics C
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-c.git &&
+cd language-basics-c &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-c)

package/extensions/builtin.language-basics-c/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-c",
+  "name": "Language Basics C",
+  "description": "Provides syntax highlighting and bracket matching in C files.",
+  "languages": [
+    {
+      "id": "c",
+      "extensions": [".c"],
+      "tokenize": "src/tokenizeC.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-c/src/tokenizeC.js

@@ -0,0 +1,180 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  AfterInclude: 2,
+  InsideLineComment: 3,
+  InsideDoubleQuoteString: 4,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.AfterInclude]: 'AfterInclude',
+  [State.InsideLineComment]: 'InsideLineComment',
+  [State.InsideDoubleQuoteString]: 'InsideDoubleQuoteString',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  Whitespace: 2,
+  Punctuation: 3,
+  CurlyOpen: 6,
+  CurlyClose: 7,
+  PropertyColon: 8,
+  VariableName: 10,
+  None: 57,
+  Unknown: 881,
+  Numeric: 883,
+  NewLine: 884,
+  Include: 885,
+  Import: 886,
+  Keyword: 887,
+  Comment: 888,
+  PunctuationString: 889,
+  String: 890,
+  PlainText: 891,
+}
+
+export const TokenMap = {
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.CurlyOpen]: 'Punctuation',
+  [TokenType.PropertyColon]: 'Punctuation',
+  [TokenType.VariableName]: 'VariableName',
+  [TokenType.None]: 'None',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Include]: 'Include',
+  [TokenType.Import]: 'Import',
+  [TokenType.Keyword]: 'Keyword',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.String]: 'String',
+  [TokenType.PlainText]: 'PlainText',
+}
+
+const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.*/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_INCLUDE = /^#include\b/
+const RE_IMPORT = /^<[^>]*>/
+const RE_KEYWORD =
+  /^(?:while|volatile|void|unsigned|union|typedef|switch|struct|static|sizeof|signed|short|return|register|long|int|if|goto|for|float|extern|enum|else|double|do|default|continue|const|char|case|break|auto)\b/
+const RE_LINE_COMMENT_START = /^\/\//
+const RE_ANYTHING_UNTIL_END = /^.+/s
+const RE_VARIABLE_NAME = /^[a-zA-Z][a-zA-Z\d\_]*/
+const RE_PUNCTUATION = /^[\(\)=\+\-><\.:\/\{\};,\[\]\*]/
+const RE_DOUBLE_QUOTE = /^"/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  tokens: [],
+}
+
+export const isEqualLineState = (lineStateA, lineStateB) => {
+  return lineStateA.state === lineStateB.state
+}
+
+/**
+ * @param {string} line
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_KEYWORD))) {
+          token = TokenType.Keyword
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_INCLUDE))) {
+          token = TokenType.Include
+          state = State.AfterInclude
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_PUNCTUATION))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.PlainText
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterInclude:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterInclude
+        } else if ((next = part.match(RE_IMPORT))) {
+          token = TokenType.Import
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideDoubleQuoteString:
+        if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        throw new Error('no')
+    }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
+  }
+  return {
+    state,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-editorconfig/README.md

@@ -0,0 +1,14 @@
+# Language Basics Editorconfig
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-editorconfig.git &&
+cd language-basics-editorconfig &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-editorconfig)

package/extensions/builtin.language-basics-editorconfig/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-editorconfig",
+  "name": "Language Basics Editorconfig",
+  "description": "Provides syntax highlighting and bracket matching in Editorconfig files.",
+  "languages": [
+    {
+      "id": "editorconfig",
+      "extensions": [".editorconfig"],
+      "tokenize": "src/tokenizeEditorConfig.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-editorconfig/src/tokenizeEditorConfig.js

@@ -0,0 +1,122 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideLineComment: 2,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.InsideLineComment]: 'InsideLineComment',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  CssSelector: 1,
+  Whitespace: 2,
+  None: 57,
+  Unknown: 881,
+  NewLine: 884,
+  Comment: 885,
+  Query: 886,
+  Text: 887,
+}
+
+export const TokenMap = {
+  [TokenType.CssSelector]: 'CssSelector',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.None]: 'None',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Query]: 'Query',
+  [TokenType.Text]: 'Text',
+}
+
+const RE_LINE_COMMENT_START = /^#/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.*/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_BLOCK_COMMENT_START = /^\/\*/
+const RE_BLOCK_COMMENT_END = /^\*\//
+const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/|$)/s
+const RE_ROUND_OPEN = /^\(/
+const RE_ROUND_CLOSE = /^\)/
+const RE_PSEUDO_SELECTOR_CONTENT = /^[^\)]+/
+const RE_SQUARE_OPEN = /^\[/
+const RE_SQUARE_CLOSE = /^\]/
+const RE_ATTRIBUTE_SELECTOR_CONTENT = /^[^\]]+/
+const RE_QUERY = /^@[a-z\-]+/
+const RE_STAR = /^\*/
+const RE_QUERY_NAME = /^[a-z\-]+/
+const RE_QUERY_CONTENT = /^[^\)]+/
+const RE_COMBINATOR = /^[\+\>\~]/
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  tokens: [],
+  stack: [],
+}
+
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        console.log({ state, line })
+        throw new Error('no')
+    }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
+  }
+  return {
+    state,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-html/src/tokenizeHtml.js

@@ -13,7 +13,8 @@ const State = {
   InsideAttributeDoubleQuote: 9,
   InsideBlockComment: 10,
   None: 11,
-  InsideString: 12,
+  InsideDoubleQuoteString: 12,
+  InsideSingleQuoteString: 13,
 }
 
 export const StateMap = {}
@@ -52,43 +53,57 @@ export const TokenMap = {
   [TokenType.PunctuationString]: 'PunctuationString',
 }
 
-const RE_WHITESPACE = /^\s+/
-const RE_DOUBLE_QUOTE = /^"/
-const RE_NEWLINE_WHITESPACE = /^\n\s*/
-const RE_BLOCK_COMMENT_START = /^<!--/
-const RE_BLOCK_COMMENT_CONTENT = /^.(?:.*?)(?=-->|$)/s
-const RE_BLOCK_COMMENT_END = /^-->/
-const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
 const RE_ANGLE_BRACKET_CLOSE = /^>/
-const RE_TAGNAME = /^[!\w]+/
-const RE_TEXT = /^[^<>\n]+/
+const RE_ANGLE_BRACKET_ONLY = /^</
+const RE_ANGLE_BRACKET_OPEN = /^</
+const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_SLASH = /^\//
 const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
+const RE_BLOCK_COMMENT_CONTENT = /^.(?:.*?)(?=-->|$)/s
+const RE_BLOCK_COMMENT_END = /^-->/
+const RE_BLOCK_COMMENT_START = /^<!--/
+const RE_DASH_DASH = /^\-\-/
+const RE_DOUBLE_QUOTE = /^"/
 const RE_EQUAL_SIGN = /^=/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
-const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
-const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
+const RE_EXCLAMATION_MARK = /^!/
 const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
-const RE_PUNCTUATION = /^[<;'".,]/
-const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
-const RE_WORD = /^[^\s]+/
-const RE_ANGLE_BRACKET_ONLY = /^</
+const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
 const RE_NEWLINE = /^\n/
-const RE_EXCLAMATION_MARK = /^!/
+const RE_NEWLINE_WHITESPACE = /^\n\s*/
+const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
+const RE_PUNCTUATION = /^[<;'".,]/
+const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
 const RE_SELF_CLOSING = /^\/>/
-const RE_DASH_DASH = /^\-\-/
+const RE_SINGLE_QUOTE = /^'/
+const RE_SLASH = /^\//
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
 const RE_TAG_TEXT = /^[^\s>]+/
-const RE_ANGLE_BRACKET_OPEN = /^</
+const RE_TAGNAME = /^[!\w]+/
+const RE_TEXT = /^[^<>\n]+/
+const RE_WHITESPACE = /^\s+/
+const RE_WORD = /^[^\s]+/
 
 export const initialLineState = {
   state: State.TopLevelContent,
 }
 
+/**
+ *
+ * @param {any} lineStateA
+ * @param {any} lineStateB
+ * @returns
+ */
 export const isLineStateEqual = (lineStateA, lineStateB) => {
   return lineStateA.state === lineStateB.state
 }
 
+/**
+ *
+ * @param {string} line
+ * @param {any} lineState
+ * @returns
+ */
 export const tokenizeLine = (line, lineState) => {
   let next = null
   let index = 0
@@ -205,7 +220,10 @@ export const tokenizeLine = (line, lineState) => {
           state = State.AfterAttributeEqualSign
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
         } else if ((next = part.match(RE_TAG_TEXT))) {
           token = TokenType.Text
           state = State.InsideOpeningTag
@@ -221,7 +239,10 @@ export const tokenizeLine = (line, lineState) => {
       case State.AfterAttributeEqualSign:
         if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
@@ -229,13 +250,24 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
-      case State.InsideString:
+      case State.InsideDoubleQuoteString:
         if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.InsideOpeningTag
         } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
           token = TokenType.String
-          state = State.InsideString
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
         } else {
           throw new Error('no')
         }
@@ -266,13 +298,3 @@ export const tokenizeLine = (line, lineState) => {
     tokens,
   }
 }
-
-// tokenizeLine(initialContext, '    <h1$0></h1>') //?
-// tokenizeLine(initialContext, `<<`)
-
-// tokenizeLine(initialContext, `<h1><//`) //?
-// tokenizeLine(initialContext, `<h1 class=></h1>`) //?
-
-// tokenizeLine(initialLineState, `<h1234567 class""=></h1234567>`)
-
-tokenizeLine('<input disabled autofocus>', initialLineState) //?

package/extensions/builtin.language-basics-json/extension.json

@@ -5,7 +5,30 @@
   "languages": [
     {
       "id": "json",
-      "extensions": [".json"],
+      "extensions": [
+        ".json",
+        ".bowerrc",
+        ".jscsrc",
+        ".webmanifest",
+        ".js.map",
+        ".css.map",
+        ".ts.map",
+        ".har",
+        ".jslintrc",
+        ".jsonld",
+        ".geojson",
+        ".ipynb",
+        ".jsonc",
+        ".eslintrc",
+        ".eslintrc.json",
+        ".jsfmtrc",
+        ".jshintrc",
+        ".swcrc",
+        ".hintrc",
+        ".babelrc",
+        ".prettierrc"
+      ],
+      "filenames": ["composer.lock", ".watchmanconfig"],
       "tokenize": "src/tokenizeJson.js"
     }
   ]

package/extensions/builtin.language-basics-powershell/src/tokenizePowershell.js

@@ -1,33 +1,141 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideLineComment: 2,
+  InsideBlockComment: 3,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.InsideLineComment]: 'InsideLineComment',
+  [State.InsideBlockComment]: 'InsideBlockComment',
+}
+
 /**
  * @enum number
  */
 export const TokenType = {
-  Text: 0,
-  NewLine: 1,
+  CssSelector: 1,
+  Whitespace: 2,
+  None: 57,
+  Unknown: 881,
+  NewLine: 884,
+  Comment: 885,
+  Query: 886,
+  Text: 887,
 }
 
 export const TokenMap = {
+  [TokenType.CssSelector]: 'CssSelector',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.None]: 'None',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Query]: 'Query',
   [TokenType.Text]: 'Text',
 }
 
+const RE_LINE_COMMENT_START = /^#/
+const RE_BLOCK_COMMENT_START = /^<#/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING_UNTIL_END = /^.+/s
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_BLOCK_COMMENT_END = /^\*\//
+const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/|$)/s
+const RE_ROUND_OPEN = /^\(/
+const RE_ROUND_CLOSE = /^\)/
+const RE_PSEUDO_SELECTOR_CONTENT = /^[^\)]+/
+const RE_SQUARE_OPEN = /^\[/
+const RE_SQUARE_CLOSE = /^\]/
+const RE_ATTRIBUTE_SELECTOR_CONTENT = /^[^\]]+/
+const RE_QUERY = /^@[a-z\-]+/
+const RE_STAR = /^\*/
+const RE_QUERY_NAME = /^[a-z\-]+/
+const RE_QUERY_CONTENT = /^[^\)]+/
+const RE_COMBINATOR = /^[\+\>\~]/
+
 export const initialLineState = {
-  state: 1,
+  state: State.TopLevelContent,
+  tokens: [],
+  stack: [],
 }
 
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
 export const tokenizeLine = (line, lineState) => {
-  if (line.length === 0) {
-    return {
-      state: 1,
-      tokens: [],
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBlockComment:
+        if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_BLOCK_COMMENT_END))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        console.log({ state, line })
+        throw new Error('no')
     }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
   }
   return {
-    state: 1,
-    tokens: [
-      {
-        type: TokenType.Text,
-        length: line.length,
-      },
-    ],
+    state,
+    tokens,
   }
 }

package/extensions/builtin.language-basics-toml/README.md

@@ -0,0 +1,14 @@
+# Language Basics Toml
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-toml.git &&
+cd language-basics-toml &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-toml)

package/extensions/builtin.language-basics-toml/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-toml",
+  "name": "Language Basics Toml",
+  "description": "Provides syntax highlighting and bracket matching in Toml files.",
+  "languages": [
+    {
+      "id": "toml",
+      "extensions": [".toml"],
+      "tokenize": "src/tokenizeToml.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-toml/src/tokenizeToml.js

@@ -0,0 +1,122 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideLineComment: 2,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.InsideLineComment]: 'InsideLineComment',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  CssSelector: 1,
+  Whitespace: 2,
+  None: 57,
+  Unknown: 881,
+  NewLine: 884,
+  Comment: 885,
+  Query: 886,
+  Text: 887,
+}
+
+export const TokenMap = {
+  [TokenType.CssSelector]: 'CssSelector',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.None]: 'None',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Query]: 'Query',
+  [TokenType.Text]: 'Text',
+}
+
+const RE_LINE_COMMENT_START = /^#/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.*/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_BLOCK_COMMENT_START = /^\/\*/
+const RE_BLOCK_COMMENT_END = /^\*\//
+const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/|$)/s
+const RE_ROUND_OPEN = /^\(/
+const RE_ROUND_CLOSE = /^\)/
+const RE_PSEUDO_SELECTOR_CONTENT = /^[^\)]+/
+const RE_SQUARE_OPEN = /^\[/
+const RE_SQUARE_CLOSE = /^\]/
+const RE_ATTRIBUTE_SELECTOR_CONTENT = /^[^\]]+/
+const RE_QUERY = /^@[a-z\-]+/
+const RE_STAR = /^\*/
+const RE_QUERY_NAME = /^[a-z\-]+/
+const RE_QUERY_CONTENT = /^[^\)]+/
+const RE_COMBINATOR = /^[\+\>\~]/
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  tokens: [],
+  stack: [],
+}
+
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        console.log({ state, line })
+        throw new Error('no')
+    }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
+  }
+  return {
+    state,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-xml/src/tokenizeXml.js

@@ -1,33 +1,298 @@
+/**
+ * @enum number
+ */
+const State = {
+  TopLevelContent: 1,
+  AfterOpeningAngleBracket: 2,
+  InsideOpeningTag: 3,
+  InsideOpeningTagAndHasSeenWhitespace: 4,
+  AfterClosingTagAngleBrackets: 5,
+  AfterClosingTagName: 6,
+  AfterAttributeName: 7,
+  AfterAttributeEqualSign: 8,
+  InsideAttributeDoubleQuote: 9,
+  InsideBlockComment: 10,
+  None: 11,
+  InsideDoubleQuoteString: 12,
+  InsideSingleQuoteString: 13,
+}
+
+export const StateMap = {}
+
 /**
  * @enum number
  */
 export const TokenType = {
-  Text: 0,
-  NewLine: 1,
+  None: 99999999,
+  Numeric: 30,
+  String: 50,
+  Whitespace: 0,
+  Comment: 60,
+  Text: 117,
+  PunctuationTag: 228,
+  TagName: 118,
+  AttributeName: 119,
+  Punctuation: 10,
+  Error: 141,
+  PunctuationString: 11,
+  NewLine: 891,
 }
 
 export const TokenMap = {
+  [TokenType.None]: 'None',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.String]: 'String',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.Comment]: 'Comment',
   [TokenType.Text]: 'Text',
+  [TokenType.PunctuationTag]: 'PunctuationTag',
+  [TokenType.TagName]: 'TagName',
+  [TokenType.AttributeName]: 'AttributeName',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.Error]: 'Error',
+  [TokenType.PunctuationString]: 'PunctuationString',
 }
 
+const RE_ANGLE_BRACKET_CLOSE = /^>/
+const RE_ANGLE_BRACKET_ONLY = /^</
+const RE_ANGLE_BRACKET_OPEN = /^</
+const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
+const RE_ANY_TEXT = /^[^\n]+/
+const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
+const RE_BLOCK_COMMENT_CONTENT = /^.(?:.*?)(?=-->|$)/s
+const RE_BLOCK_COMMENT_END = /^-->/
+const RE_BLOCK_COMMENT_START = /^<!--/
+const RE_DASH_DASH = /^\-\-/
+const RE_DOUBLE_QUOTE = /^"/
+const RE_EQUAL_SIGN = /^=/
+const RE_EXCLAMATION_MARK = /^!/
+const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
+const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
+const RE_NEWLINE = /^\n/
+const RE_NEWLINE_WHITESPACE = /^\n\s*/
+const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
+const RE_PUNCTUATION = /^[<;'".,]/
+const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
+const RE_SELF_CLOSING = /^\/>/
+const RE_SINGLE_QUOTE = /^'/
+const RE_SLASH = /^\//
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_TAG_TEXT = /^[^\s>]+/
+const RE_TAGNAME = /^[!\w]+/
+const RE_TEXT = /^[^<>\n]+/
+const RE_WHITESPACE = /^\s+/
+const RE_WORD = /^[^\s]+/
+
 export const initialLineState = {
-  state: 1,
+  state: State.TopLevelContent,
 }
 
+/**
+ *
+ * @param {any} lineStateA
+ * @param {any} lineStateB
+ * @returns
+ */
+export const isLineStateEqual = (lineStateA, lineStateB) => {
+  return lineStateA.state === lineStateB.state
+}
+
+/**
+ *
+ * @param {string} line
+ * @param {any} lineState
+ * @returns
+ */
 export const tokenizeLine = (line, lineState) => {
-  if (line.length === 0) {
-    return {
-      state: 1,
-      tokens: [],
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_ANGLE_BRACKET_OPEN_TAG))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterOpeningAngleBracket
+        } else if ((next = part.match(RE_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterOpeningAngleBracket:
+        if ((next = part.match(RE_TAGNAME))) {
+          token = TokenType.TagName
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_SLASH))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterClosingTagAngleBrackets
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANY_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideOpeningTag:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EXCLAMATION_MARK))) {
+          token = TokenType.PunctuationTag
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.InsideOpeningTagAndHasSeenWhitespace
+        } else if ((next = part.match(RE_DASH_DASH))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.InsideOpeningTag
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterClosingTagAngleBrackets:
+        if ((next = part.match(RE_TAGNAME))) {
+          token = TokenType.TagName
+          state = State.AfterClosingTagName
+        } else if ((next = part.match(RE_ANY_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterClosingTagName:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideOpeningTagAndHasSeenWhitespace:
+        if ((next = part.match(RE_ATTRIBUTE_NAME))) {
+          token = TokenType.AttributeName
+          state = State.AfterAttributeName
+        } else if ((next = part.match(RE_SELF_CLOSING))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterAttributeName:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EQUAL_SIGN))) {
+          token = TokenType.Punctuation
+          state = State.AfterAttributeEqualSign
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.InsideOpeningTagAndHasSeenWhitespace
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterAttributeEqualSign:
+        if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideDoubleQuoteString:
+        if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBlockComment:
+        if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_BLOCK_COMMENT_END))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        state
+        throw new Error('no')
     }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
   }
   return {
-    state: 1,
-    tokens: [
-      {
-        type: TokenType.Text,
-        length: line.length,
-      },
-    ],
+    state,
+    tokens,
   }
 }

package/extensions/builtin.language-basics-yaml/extension.json

@@ -5,7 +5,7 @@
   "languages": [
     {
       "id": "yaml",
-      "extensions": [".yml"],
+      "extensions": [".yml", ".eyaml", ".eyml", ".yaml", ".cff"],
       "tokenize": "src/tokenizeYaml.js"
     }
   ]

package/extensions/builtin.language-basics-yaml/src/tokenizeYaml.js

@@ -1,33 +1,122 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideLineComment: 2,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.InsideLineComment]: 'InsideLineComment',
+}
+
 /**
  * @enum number
  */
 export const TokenType = {
-  Text: 0,
-  NewLine: 1,
+  CssSelector: 1,
+  Whitespace: 2,
+  None: 57,
+  Unknown: 881,
+  NewLine: 884,
+  Comment: 885,
+  Query: 886,
+  Text: 887,
 }
 
 export const TokenMap = {
+  [TokenType.CssSelector]: 'CssSelector',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.None]: 'None',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Query]: 'Query',
   [TokenType.Text]: 'Text',
 }
 
+const RE_LINE_COMMENT_START = /^#/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.*/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_BLOCK_COMMENT_START = /^\/\*/
+const RE_BLOCK_COMMENT_END = /^\*\//
+const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/|$)/s
+const RE_ROUND_OPEN = /^\(/
+const RE_ROUND_CLOSE = /^\)/
+const RE_PSEUDO_SELECTOR_CONTENT = /^[^\)]+/
+const RE_SQUARE_OPEN = /^\[/
+const RE_SQUARE_CLOSE = /^\]/
+const RE_ATTRIBUTE_SELECTOR_CONTENT = /^[^\]]+/
+const RE_QUERY = /^@[a-z\-]+/
+const RE_STAR = /^\*/
+const RE_QUERY_NAME = /^[a-z\-]+/
+const RE_QUERY_CONTENT = /^[^\)]+/
+const RE_COMBINATOR = /^[\+\>\~]/
+
 export const initialLineState = {
-  state: 1,
+  state: State.TopLevelContent,
+  tokens: [],
+  stack: [],
 }
 
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
 export const tokenizeLine = (line, lineState) => {
-  if (line.length === 0) {
-    return {
-      state: 1,
-      tokens: [],
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        console.log({ state, line })
+        throw new Error('no')
     }
+    index += next[0].length
+    tokens.push({
+      type: token,
+      length: next[0].length,
+    })
   }
   return {
-    state: 1,
-    tokens: [
-      {
-        type: TokenType.Text,
-        length: line.length,
-      },
-    ],
+    state,
+    tokens,
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/shared-process",
-  "version": "0.0.37",
+  "version": "0.0.38",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -38,8 +38,8 @@
     "verror": "^1.10.1",
     "ws": "^8.8.1",
     "xdg-basedir": "^5.1.0",
-    "@lvce-editor/pty-host": "0.0.37",
-    "@lvce-editor/extension-host": "0.0.37"
+    "@lvce-editor/pty-host": "0.0.38",
+    "@lvce-editor/extension-host": "0.0.38"
   },
   "optionalDependencies": {
     "electron-clipboard-ex": "^1.3.3",