@lvce-editor/shared-process 0.11.8 → 0.11.10

package/extensions/builtin.language-basics-python/src/tokenizePython.js

@@ -16,6 +16,8 @@ export const State = {
   InsideDoubleQuoteString: 11,
   InsideLineComment: 12,
   InsideSingleQuoteString: 13,
+  InsideTripleDoubleQuoteString: 14,
+  InsideTripleSingleQuoteString: 15,
 }
 
 /**
@@ -69,19 +71,29 @@ export const TokenMap = {
 
 const RE_WHITESPACE = /^\s+/
 const RE_DOUBLE_QUOTE = /^"/
+const RE_TRIPLE_DOUBLE_QUOTE = /^"{3}/
 const RE_SINGLE_QUOTE = /^'/
 const RE_TEXT = /^.+/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
-const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^'\\]+/
+const RE_STRING_TRIPLE_DOUBLE_QUOTE_CONTENT = /^.+(?=""")/
 const RE_LINE_COMMENT = /^#/
 const RE_KEYWORD =
   /^(?:__debug__|Ellipsis|yield|with|while|try|true|True|return|raise|pass|or|NotImplemented|not|nonlocal|None|lambda|is|in|import|if|global|from|for|finally|False|false|except|else|elif|del|def|continue|class|break|await|async|assert|as|and)\b/
 const RE_VARIABLE_NAME = /^[_a-zA-Z][a-zA-Z\d_]*/
 const RE_NUMERIC =
   /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
-const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%]+/
+const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%\|\~]+/
 const RE_FUNCTION_CALL_NAME = /^[\w]+(?=\s*\()/
+const RE_DECORATOR = /^@[\w]+/
+const RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES = /.*(?=""")/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES = /.*(?=''')/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON = /.*/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_BACKSLASH = /^\\/
+const RE_TRIPLE_SINGLE_QUOTE = /^'{3}/
+const RE_NUMERIC_OCTAL = /^0(?:o|O)?[0-7][0-7_]*(n)?\b/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -130,6 +142,14 @@ export const tokenizeLine = (line, lineState) => {
             case 'finally':
             case 'while':
             case 'for':
+            case 'continue':
+            case 'not':
+            case 'except':
+            case 'with':
+            case 'as':
+            case 'raise':
+            case 'break':
+            case 'pass':
               token = TokenType.KeywordControl
               break
             default:
@@ -149,6 +169,15 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC_OCTAL))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleDoubleQuoteString
+        } else if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleSingleQuoteString
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.InsideDoubleQuoteString
@@ -158,6 +187,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.InsideLineComment
+        } else if ((next = part.match(RE_DECORATOR))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -173,6 +205,12 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
         } else {
           throw new Error('no')
         }
@@ -184,6 +222,49 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_SINGLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleDoubleQuoteString:
+        if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleSingleQuoteString:
+        if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          next
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
         } else {
           throw new Error('no')
         }
@@ -209,3 +290,5 @@ export const tokenizeLine = (line, lineState) => {
     tokens,
   }
 }
+
+tokenizeLine(`'''test'''`, initialLineState) //?

package/extensions/builtin.language-basics-ruby/README.md

@@ -0,0 +1,14 @@
+# Language Basics Ruby
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-ruby.git &&
+cd language-basics-ruby &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-ruby)

package/extensions/builtin.language-basics-ruby/extension.json

@@ -0,0 +1,48 @@
+{
+  "id": "builtin.language-basics-ruby",
+  "name": "Language Basics Ruby",
+  "description": "Provides syntax highlighting and bracket matching in Ruby files.",
+  "languages": [
+    {
+      "id": "ruby",
+      "extensions": [
+        ".rb",
+        ".rbx",
+        ".rjs",
+        ".gemspec",
+        ".rake",
+        ".ru",
+        ".erb",
+        ".podspec",
+        ".rbi"
+      ],
+      "fileNames": [
+        "rakefile",
+        "gemfile",
+        "guardfile",
+        "podfile",
+        "capfile",
+        "cheffile",
+        "hobofile",
+        "vagrantfile",
+        "appraisals",
+        "rantfile",
+        "berksfile",
+        "berksfile.lock",
+        "thorfile",
+        "puppetfile",
+        "dangerfile",
+        "brewfile",
+        "fastfile",
+        "appfile",
+        "deliverfile",
+        "matchfile",
+        "scanfile",
+        "snapfile",
+        "gymfile"
+      ],
+      "firstLine": "^#!\\s*/.*\\bruby\\b",
+      "tokenize": "src/tokenizeRuby.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-ruby/src/tokenizeRuby.js

@@ -0,0 +1,241 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideDoubleQuoteString: 2,
+  InsideSingleQuoteString: 3,
+  InsideBackTickQuoteString: 4,
+  InsideEof: 5,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  None: 1,
+  Whitespace: 2,
+  PunctuationString: 3,
+  String: 4,
+  Keyword: 5,
+  Numeric: 6,
+  Punctuation: 7,
+  VariableName: 8,
+  Comment: 9,
+  KeywordControl: 10,
+  LanguageConstant: 11,
+  KeywordReturn: 12,
+  Regex: 13,
+  KeywordImport: 14,
+}
+
+export const TokenMap = {
+  [TokenType.None]: 'None',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.String]: 'String',
+  [TokenType.Keyword]: 'Keyword',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.VariableName]: 'VariableName',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
+  [TokenType.Regex]: 'Regex',
+  [TokenType.KeywordImport]: 'KeywordImport',
+}
+
+const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.+/s
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_QUOTE_DOUBLE = /^"/
+const RE_QUOTE_SINGLE = /^'/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^'\\]+/
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
+const RE_KEYWORD =
+  /^(?:alias|and|begin|break|case|class|def|defined\?|do|else|elsif|end|ensure|false|for|if|in|module|next|nil|not|or|redo|require|rescue|retry|return|self|super|then|true|undef|unless|until|when|while|yield)\b/
+const RE_VARIABLE_NAME = /^[a-zA-Z\_\$]+/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\-\|\&\+\?\!\%\*\/\@\~]/
+const RE_NUMERIC = /^\d+/
+const RE_LINE_COMMENT = /^#.*/s
+const RE_EOF_START = /^<<\-?\s*([\w\!]+)/
+const RE_EOF_CONTENT = /.*/s
+// const RE_REGEX = /^\/.+/
+const RE_REGEX = /^\/.*\//
+const RE_STRING_ESCAPE = /^\\./
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  tokens: [],
+  stringEnd: '',
+}
+
+export const isEqualLineState = (lineStateA, lineStateB) => {
+  return lineStateA.state === lineStateB.state
+}
+
+export const hasArrayReturn = true
+
+/**
+ * @param {string} line
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  let stringEnd = lineState.stringEnd
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_KEYWORD))) {
+          switch (next[0]) {
+            case 'break':
+            case 'case':
+            case 'do':
+            case 'elsif':
+            case 'else':
+            case 'for':
+            case 'if':
+            case 'in':
+            case 'then':
+            case 'while':
+            case 'end':
+            case 'begin':
+            case 'rescue':
+            case 'unless':
+              token = TokenType.KeywordControl
+              break
+            case 'true':
+            case 'false':
+            case 'nil':
+              token = TokenType.LanguageConstant
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'require':
+              token = TokenType.KeywordImport
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EOF_START))) {
+          token = TokenType.Punctuation
+          state = State.InsideEof
+          stringEnd = next[1]
+        } else if ((next = part.match(RE_REGEX))) {
+          token = TokenType.Regex
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_PUNCTUATION))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.InsideBackTickQuoteString
+        } else if ((next = part.match(RE_LINE_COMMENT))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          console.log({ part })
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideDoubleQuoteString:
+        if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBackTickQuoteString:
+        if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideBackTickQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideEof:
+        if ((next = part.match(stringEnd))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EOF_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideEof
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        throw new Error('no')
+    }
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
+  }
+  return {
+    state,
+    tokens,
+    stringEnd,
+  }
+}

package/extensions/builtin.language-basics-scss/README.md

@@ -0,0 +1,14 @@
+# Language Basics Scss
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-scss.git &&
+cd language-basics-scss &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-scss)

package/extensions/builtin.language-basics-scss/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-scss",
+  "name": "Language Basics Scss",
+  "description": "Provides syntax highlighting and bracket matching in Scss files.",
+  "languages": [
+    {
+      "id": "scss",
+      "extensions": [".scss"],
+      "tokenize": "src/tokenizeScss.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-scss/src/tokenizeScss.js

@@ -0,0 +1,28 @@
+/**
+ * @enum number
+ */
+export const TokenType = {
+  Text: 1,
+}
+
+export const TokenMap = {
+  [TokenType.Text]: 'Text',
+}
+
+export const initialLineState = {
+  state: 1,
+  tokens: [],
+}
+
+export const hasArrayReturn = true
+
+/**
+ * @param {string} line
+ */
+export const tokenizeLine = (line) => {
+  const tokens = [TokenType.Text, line.length]
+  return {
+    state: 1,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-shellscript/extension.json

@@ -46,7 +46,8 @@
         "zprofile",
         "zlogout",
         "bashrc_Apple_Terminal",
-        "zshrc_Apple_Terminal"
+        "zshrc_Apple_Terminal",
+        "xvfb.init"
       ],
       "firstLine": "^#!.*\\b(bash|fish|zsh|sh|ksh|dtksh|pdksh|mksh|ash|dash|yash|sh|csh|jcsh|tcsh|itcsh).*|^#\\s*-\\*-[^*]*mode:\\s*shell-script[^*]*-\\*-",
       "tokenize": "src/tokenizeShellScript.js"