@lvce-editor/shared-process 0.11.12 → 0.12.1

package/extensions/builtin.language-basics-html/src/tokenizeHtml.js

@@ -15,6 +15,8 @@ const State = {
   None: 11,
   InsideDoubleQuoteString: 12,
   InsideSingleQuoteString: 13,
+  InsideScriptContent: 14,
+  InsideStyleContent: 15,
 }
 
 export const StateMap = {}
@@ -36,6 +38,7 @@ export const TokenType = {
   Error: 141,
   PunctuationString: 11,
   NewLine: 891,
+  Embedded: 999,
 }
 
 export const TokenMap = {
@@ -51,18 +54,20 @@ export const TokenMap = {
   [TokenType.Punctuation]: 'Punctuation',
   [TokenType.Error]: 'Error',
   [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.Embedded]: 'Embedded',
 }
 
 const RE_ANGLE_BRACKET_CLOSE = /^>/
 const RE_ANGLE_BRACKET_ONLY = /^</
 const RE_ANGLE_BRACKET_OPEN = /^</
-const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
+const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?![\s!\%])/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
+const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-\:]+/
 const RE_BLOCK_COMMENT_CONTENT_1 = /^.+?(?=-->)/s
 const RE_BLOCK_COMMENT_CONTENT_2 = /^.+$/s
 const RE_BLOCK_COMMENT_END = /^-->/
 const RE_BLOCK_COMMENT_START = /^<!--/
+const RE_DOCTYPE = /^<!(?=\w)/
 const RE_DASH_DASH = /^\-\-/
 const RE_DOUBLE_QUOTE = /^"/
 const RE_EQUAL_SIGN = /^=/
@@ -80,14 +85,24 @@ const RE_SLASH = /^\//
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
 const RE_TAG_TEXT = /^[^\s>]+/
-const RE_TAGNAME = /^[!\w]+/
+const RE_TAGNAME = /^[!\w\:]+/
 const RE_TEXT = /^[^<>\n]+/
 const RE_WHITESPACE = /^\s+/
 const RE_WORD = /^[^\s]+/
-const RE_ATTRIBUTE_VALUE_UNQUOTED = /^[^<>]+/
+const RE_ATTRIBUTE_VALUE_UNQUOTED = /^[^<>\s]+/
+const RE_SCRIPT_CONTENT = /^..*?(?=(?:<\/script)|$)/s
+const RE_SCRIPT_CONTENT_END = /^<\/script/
+const RE_STYLE_CONTENT = /^..*?(?=(?:<\/style)|$)/s
+const RE_STYLE_CONTENT_END = /^<\/style/
 
 export const initialLineState = {
   state: State.TopLevelContent,
+  tag: '',
+  stack: [],
+  embeddedLanguage: '',
+  embeddedLanguageStart: 0,
+  embeddedLanguageEnd: 0,
+  embeddedState: undefined,
 }
 
 /**
@@ -102,6 +117,14 @@ export const isLineStateEqual = (lineStateA, lineStateB) => {
 
 export const hasArrayReturn = true
 
+const getEmbeddedScriptLanguageId = () => {
+  return 'javascript'
+}
+
+const getEmbeddedStyleLanguageId = () => {
+  return 'css'
+}
+
 /**
  *
  * @param {string} line
@@ -114,6 +137,10 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let tag = lineState.tag
+  let embeddedLanguage = lineState.embeddedLanguage
+  let embeddedLanguageStart = lineState.embeddedLanguageStart
+  let embeddedLanguageEnd = lineState.embeddedLanguageEnd
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -124,9 +151,15 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.Text
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_DOCTYPE))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterOpeningAngleBracket
         } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -139,6 +172,7 @@ export const tokenizeLine = (line, lineState) => {
         if ((next = part.match(RE_TAGNAME))) {
           token = TokenType.TagName
           state = State.InsideOpeningTag
+          tag = next[0]
         } else if ((next = part.match(RE_SLASH))) {
           token = TokenType.PunctuationTag
           state = State.AfterClosingTagAngleBrackets
@@ -160,6 +194,19 @@ export const tokenizeLine = (line, lineState) => {
         if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
+          switch (tag) {
+            case 'script':
+              state = State.InsideScriptContent
+              embeddedLanguage = getEmbeddedScriptLanguageId()
+              embeddedLanguageStart = index + next[0].length
+              break
+            case 'style':
+              state = State.InsideStyleContent
+              embeddedLanguage = getEmbeddedStyleLanguageId()
+              embeddedLanguageStart = index + next[0].length
+            default:
+              break
+          }
         } else if ((next = part.match(RE_EXCLAMATION_MARK))) {
           token = TokenType.PunctuationTag
           state = State.InsideOpeningTag
@@ -169,6 +216,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_DASH_DASH))) {
           token = TokenType.Comment
           state = State.InsideBlockComment
+        } else if ((next = part.match(RE_PUNCTUATION_SELF_CLOSING))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_TAG_TEXT))) {
           token = TokenType.Text
           state = State.InsideOpeningTag
@@ -213,6 +263,10 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
+          if (tag === 'script') {
+            state = State.InsideScriptContent
+            embeddedLanguage = 'javascript'
+          }
         } else if ((next = part.match(RE_TAG_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -301,6 +355,43 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideScriptContent:
+        if ((next = part.match(RE_SCRIPT_CONTENT_END))) {
+          token = TokenType.TagName
+          state = State.AfterClosingTagName
+          tokens.push(TokenType.PunctuationTag, 2, TokenType.TagName, 6)
+          index += next[0].length
+          embeddedLanguage = ''
+          continue
+        } else if ((next = part.match(RE_SCRIPT_CONTENT))) {
+          token = TokenType.Embedded
+          state = State.InsideScriptContent
+          embeddedLanguageStart = index
+          embeddedLanguageEnd = index + next[0].length
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
+      case State.InsideStyleContent:
+        if ((next = part.match(RE_STYLE_CONTENT_END))) {
+          token = TokenType.TagName
+          state = State.AfterClosingTagName
+          tokens.push(TokenType.PunctuationTag, 2, TokenType.TagName, 6)
+          embeddedLanguageEnd = index
+          index += next[0].length
+          embeddedLanguage = ''
+          continue
+        } else if ((next = part.match(RE_STYLE_CONTENT))) {
+          token = TokenType.Embedded
+          state = State.InsideStyleContent
+          embeddedLanguageStart = index
+          embeddedLanguageEnd = index + next[0].length
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
       default:
         state
         throw new Error('no')
@@ -316,5 +407,9 @@ export const tokenizeLine = (line, lineState) => {
   return {
     state,
     tokens,
+    tag,
+    embeddedLanguage,
+    embeddedLanguageStart,
+    embeddedLanguageEnd,
   }
 }

package/extensions/builtin.language-basics-javascript/src/tokenizeJavaScript.js

@@ -52,6 +52,7 @@ export const TokenType = {
   FunctionName: 885,
   KeywordThis: 886,
   KeywordOperator: 887,
+  KeywordFunction: 889,
 }
 
 export const TokenMap = {
@@ -80,10 +81,11 @@ export const TokenMap = {
   [TokenType.FunctionName]: 'Function',
   [TokenType.KeywordThis]: 'KeywordThis',
   [TokenType.KeywordOperator]: 'KeywordOperator',
+  [TokenType.KeywordFunction]: 'KeywordFunction',
 }
 
 const RE_KEYWORD =
-  /^(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|false|finally|for|from|function|if|implements|import|in|instanceof|interface|let|new|null|package|private|protected|public|return|super|switch|static|this|throw|try|true|typeof|var|void|while|with|yield)\b/
+  /^(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|false|finally|for|from|function|if|implements|import|in|Infinity|instanceof|interface|let|new|null|of|package|private|protected|public|return|super|switch|static|this|throw|try|true|typeof|undefined|var|void|while|with|yield)\b/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
 const RE_SQUARE_OPEN = /^\[/
@@ -118,20 +120,29 @@ const RE_QUOTE_DOUBLE = /^"/
 const RE_QUOTE_BACKTICK = /^`/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^'\\]+/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
-const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`\\]+/
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`\\\$]+/
 const RE_STRING_ESCAPE = /^\\./
 const RE_SHEBANG = /^#!.*/
 const RE_FUNCTION_CALL_NAME = /^[\w]+(?=\s*(\(|\=\s*function|\=\s*\())/
 const RE_DECORATOR = /^@\w+/
+const RE_BACKSLASH = /^\\/
+const RE_DOLLAR_CURLY_OPEN = /^\$\{/
+const RE_DOLLAR = /^\$/
+const RE_ANYTHING = /^.+/u
 
 // copied from https://github.com/PrismJS/prism/blob/master/components/prism-javascript.js#L57
 const RE_REGEX =
   /^((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/
 
 const RE_VARIABLE_NAME_SPECIAL = /\p{L}/u
+const RE_VARIABLE_NAME_SPECIAL_2 = /./u
 
 export const initialLineState = {
   state: State.TopLevelContent,
+  /**
+   * @type {any[]}
+   */
+  stack: [],
 }
 
 export const hasArrayReturn = true
@@ -148,6 +159,7 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let stack = [...lineState.stack]
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -160,6 +172,7 @@ export const tokenizeLine = (line, lineState) => {
             case 'true':
             case 'false':
             case 'null':
+            case 'undefined':
               token = TokenType.LanguageConstant
               break
             case 'import':
@@ -201,6 +214,16 @@ export const tokenizeLine = (line, lineState) => {
             case 'in':
               token = TokenType.KeywordOperator
               break
+            case 'function':
+              token = TokenType.KeywordFunction
+              break
+            case 'Infinity':
+              token = TokenType.Numeric
+              state = State.TopLevelContent
+              break
+            case 'of':
+              token = TokenType.KeywordOperator
+              break
             default:
               token = TokenType.Keyword
               break
@@ -215,10 +238,19 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop() || State.TopLevelContent
+        } else if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          stack.push(State.TopLevelContent)
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
           if (next[0] === '.') {
             state = State.AfterPropertyDot
+          } else if (next[0] === '}') {
+            state = stack.pop() || State.TopLevelContent
           } else {
             state = State.TopLevelContent
           }
@@ -258,6 +290,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_VARIABLE_NAME_SPECIAL))) {
           token = TokenType.VariableName
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else {
           throw new Error('no')
         }
@@ -306,7 +341,11 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_STRING_ESCAPE))) {
           token = TokenType.String
           state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
         } else {
+          part
           throw new Error('no')
         }
         break
@@ -320,6 +359,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_STRING_ESCAPE))) {
           token = TokenType.String
           state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
         } else {
           throw new Error('no')
         }
@@ -331,9 +373,19 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
           token = TokenType.String
           state = State.InsideBacktickString
+        } else if ((next = part.match(RE_DOLLAR_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+          stack.push(State.InsideBacktickString)
         } else if ((next = part.match(RE_STRING_ESCAPE))) {
           token = TokenType.String
           state = State.InsideBacktickString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_DOLLAR))) {
+          token = TokenType.String
+          state = State.InsideBacktickString
         } else {
           throw new Error('no')
         }
@@ -357,6 +409,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_VARIABLE_NAME_SPECIAL_2))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else {
           throw new Error('no')
         }
@@ -372,8 +427,12 @@ export const tokenizeLine = (line, lineState) => {
   if (state === State.InsideLineComment) {
     state = State.TopLevelContent
   }
+  if (state === State.InsideSingleQuoteString && !line.endsWith('\\')) {
+    state = State.TopLevelContent
+  }
   return {
     state,
     tokens,
+    stack,
   }
 }

package/extensions/builtin.language-basics-json/extension.json

@@ -43,7 +43,12 @@
         ".publishrc",
         ".whitesource",
         ".mention-bot",
-        "pdfjs.config"
+        "pdfjs.config",
+        "tsconfig.tsbuildinfo",
+        ".gdntsa",
+        ".nycrc",
+        "manifest.webapp",
+        ".alexrc"
       ],
       "tokenize": "src/tokenizeJson.js"
     }

package/extensions/builtin.language-basics-json/src/tokenizeJson.js

@@ -11,6 +11,7 @@ export const State = {
   JsonPropertyKey: 7,
   InsideLineComment: 8,
   InsidePropertyNameString: 9,
+  InsideBlockComment: 10,
 }
 
 export const StateMap = {
@@ -97,6 +98,10 @@ const RE_TEXT = /^[^\s\{\}\[\]]+/
 // const RE_BACK_SLASH = /^\\/
 const RE_STRING_ESCAPE = /^\\.?/
 const RE_WORD = /^\w+/
+const RE_BLOCK_COMMENT_START = /^\/\*/
+const RE_BLOCK_COMMENT_CONTENT = /^.+?(?=\*\/)/
+const RE_BLOCK_COMMENT_END = /^\*\//
+const RE_ANYTHING_UNTIL_END = /^.+/s
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -158,6 +163,10 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_CURLY_CLOSE))) {
           token = TokenType.CurlyClose
           state = stack.pop()
+        } else if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+          stack.push(State.TopLevelContent)
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -186,6 +195,10 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_WORD))) {
           token = TokenType.Text
           state = State.AfterPropertyName
+        } else if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+          stack.push(State.AfterCurlyOpen)
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.AfterCurlyOpen
@@ -298,6 +311,20 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideBlockComment:
+        if ((next = part.match(RE_BLOCK_COMMENT_END))) {
+          token = TokenType.Comment
+          state = stack.pop() || State.TopLevelContent
+        } else if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         state
         throw new Error('no')

package/extensions/builtin.language-basics-jsx/README.md

@@ -0,0 +1,14 @@
+# Language Basics Jsx
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-jsx.git &&
+cd language-basics-jsx &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-jsx)

package/extensions/builtin.language-basics-jsx/extension.json

@@ -0,0 +1,13 @@
+{
+  "id": "builtin.language-basics-jsx",
+  "name": "Language Basics Jsx",
+  "description": "Provides syntax highlighting and bracket matching in Jsx files.",
+  "languages": [
+    {
+      "id": "jsx",
+      "extensions": [".jsx"],
+      "tokenize": "src/tokenizeJsx.js",
+      "configuration": "./languageConfiguration.json"
+    }
+  ]
+}