@lvce-editor/shared-process 0.11.7 → 0.11.9

package/extensions/builtin.language-basics-json5/src/tokenizeJson5.js

@@ -0,0 +1,321 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  AfterCurlyOpen: 2,
+  InsideString: 3,
+  AfterPropertyName: 4,
+  AfterPropertyNameAfterColon: 5,
+  AfterPropertyValue: 6,
+  JsonPropertyKey: 7,
+  InsideLineComment: 8,
+  InsidePropertyNameString: 9,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+  [State.AfterCurlyOpen]: 'AfterCurlyOpen',
+  [State.InsideString]: 'InsideString',
+  [State.AfterPropertyName]: 'AfterPropertyName',
+  [State.AfterPropertyNameAfterColon]: 'AfterPropertyNameAfterColon',
+  [State.AfterPropertyValue]: 'AfterPropertyValue',
+  [State.JsonPropertyKey]: 'JsonPropertyKey',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  CssSelector: 1,
+  Whitespace: 2,
+  Punctuation: 3,
+  PropertyName: 4,
+  PropertyValue: 5,
+  CurlyOpen: 6,
+  CurlyClose: 7,
+  PropertyColon: 8,
+  PropertySemicolon: 9,
+  String: 10,
+  LanguageConstant: 11,
+  NewLine: 12,
+  JsonPropertyName: 13,
+  PropertyValueString: 14,
+  Numeric: 15,
+  LanguageConstantBoolean: 16,
+  Comment: 17,
+  Error: 18,
+  Unknown: 19,
+  Text: 20,
+  JsonPropertyValueString: 21,
+}
+
+export const TokenMap = {
+  [TokenType.CssSelector]: 'CssSelector',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.PropertyName]: 'PropertyName',
+  [TokenType.PropertyValue]: 'PropertyValue',
+  [TokenType.CurlyOpen]: 'CurlyOpen',
+  [TokenType.CurlyClose]: 'CurlyClose',
+  [TokenType.PropertyColon]: 'PropertyColon',
+  [TokenType.PropertySemicolon]: 'PropertySemicolon',
+  [TokenType.String]: 'String',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.LanguageConstantBoolean]: 'LanguageConstantBoolean',
+  [TokenType.JsonPropertyName]: 'JsonPropertyName',
+  [TokenType.PropertyValueString]: 'PropertyValueString',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Error]: 'Error',
+  [TokenType.Unknown]: 'Unknown',
+  [TokenType.NewLine]: 'NewLine',
+  [TokenType.Text]: 'Text',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.JsonPropertyValueString]: 'JsonPropertyValueString',
+}
+
+const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
+const RE_WHITESPACE = /^\s+/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^\n;]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_DOUBLE_QUOTE = /^"/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_LANGUAGE_CONSTANT = /^(?:true|false|null)/
+const RE_SQUARE_OPEN = /^\[/
+const RE_SQUARE_CLOSE = /^\]/
+const RE_LINE_COMMENT = /^\/\/.*/
+const RE_ANYTHING = /^.+/s
+const RE_NUMERIC =
+  /^((0(x|X)[0-9a-fA-F]*)|(\-?([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
+const RE_TEXT = /^[^\s\{\}\[\]]+/
+// const RE_ESCAPED_QUOTE = /^\\"/
+// const RE_BACK_SLASH = /^\\/
+const RE_STRING_ESCAPE = /^\\.?/
+const RE_WORD = /^\w+/
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  stack: [],
+}
+
+export const hasArrayReturn = true
+
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next
+  let index = 0
+  let token
+  let state = lineState.state
+  let __r = 0
+  const tokens = []
+  const stack = lineState.stack
+  state
+  while (index < line.length) {
+    if (__r++ > 100_000_000_000) {
+      throw new Error('endless loop')
+    }
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.AfterCurlyOpen
+          stack.push(State.TopLevelContent)
+        } else if ((next = part.match(RE_SQUARE_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+          stack.push(State.TopLevelContent)
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_SQUARE_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop()
+        } else if ((next = part.match(RE_COMMA))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
+          token = TokenType.LanguageConstant
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideString
+          stack.push(State.TopLevelContent)
+        } else if ((next = part.match(RE_LINE_COMMENT))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.CurlyClose
+          state = stack.pop()
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterCurlyOpen:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterCurlyOpen
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsidePropertyNameString
+          stack.push(State.AfterPropertyName)
+        } else if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop()
+          if (!state) {
+            throw new Error('imbalanced json')
+          }
+        } else if ((next = part.match(RE_LINE_COMMENT))) {
+          token = TokenType.Comment
+          state = State.AfterCurlyOpen
+        } else if ((next = part.match(RE_WORD))) {
+          token = TokenType.PropertyName
+          state = State.AfterPropertyName
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.AfterCurlyOpen
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsidePropertyNameString:
+        if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.JsonPropertyName
+          state = State.InsidePropertyNameString
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = stack.pop()
+          state
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.JsonPropertyName
+          state = State.InsidePropertyNameString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideString:
+        if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.JsonPropertyValueString
+          state = State.InsideString
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = stack.pop()
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideString
+        } else {
+          line.slice(0, index) //?
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterPropertyName:
+        if ((next = part.match(RE_COLON))) {
+          token = TokenType.Punctuation
+          state = State.AfterPropertyNameAfterColon
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyName
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.AfterCurlyOpen
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterPropertyNameAfterColon:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyNameAfterColon
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideString
+          stack.push(State.AfterPropertyValue)
+        } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
+          token = TokenType.LanguageConstant
+          state = State.AfterPropertyValue
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.AfterPropertyValue
+        } else if ((next = part.match(RE_TEXT))) {
+          token = TokenType.Text
+          state = State.AfterCurlyOpen
+        } else if ((next = part.match(RE_CURLY_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.AfterCurlyOpen
+          stack.push(State.AfterPropertyValue)
+        } else if ((next = part.match(RE_SQUARE_OPEN))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+          stack.push(State.AfterPropertyValue)
+        } else {
+          line.slice(0, index) //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterPropertyValue:
+        if ((next = part.match(RE_CURLY_CLOSE))) {
+          token = TokenType.Punctuation
+          state = stack.pop()
+          if (!state) {
+            throw new Error('inbalanced json')
+          }
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyValue
+        } else if ((next = part.match(RE_COMMA))) {
+          token = TokenType.Punctuation
+          state = State.AfterCurlyOpen
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.AfterCurlyOpen
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        state
+        throw new Error('no')
+    }
+
+    const currentTokenText = next[0]
+    const currentTokenLength = currentTokenText.length
+    index += currentTokenLength
+    tokens.push(token, currentTokenLength)
+  }
+  if (state === State.InsideLineComment) {
+    state = State.TopLevelContent
+  }
+  if (state === State.AfterPropertyValue) {
+    state = State.AfterCurlyOpen
+  }
+  return {
+    state,
+    stack,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-perl/extension.json

@@ -6,7 +6,8 @@
     {
       "id": "perl",
       "extensions": [".pl", ".pm", ".pod", ".t", ".pL", ".psgi"],
-      "tokenize": "src/tokenizePerl.js"
+      "tokenize": "src/tokenizePerl.js",
+      "firstLine": "^#!.*\\bperl\\b"
     }
   ]
 }

package/extensions/builtin.language-basics-perl/src/tokenizePerl.js

@@ -5,6 +5,7 @@ export const State = {
   TopLevelContent: 1,
   InsideString: 2,
   InsideLineComment: 3,
+  InsideSingleQuoteString: 4,
 }
 
 export const StateMap = {
@@ -25,6 +26,9 @@ export const TokenType = {
   VariableName: 8,
   Comment: 885,
   Text: 9,
+  KeywordControl: 10,
+  KeywordReturn: 11,
+  Function: 12,
 }
 
 export const TokenMap = {
@@ -38,11 +42,14 @@ export const TokenMap = {
   [TokenType.VariableName]: 'VariableName',
   [TokenType.Comment]: 'Comment',
   [TokenType.Text]: 'Text',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
+  [TokenType.Function]: 'Function',
 }
 
 const RE_LINE_COMMENT = /^#.*/s
 const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
-const RE_WHITESPACE = /^ +/
+const RE_WHITESPACE = /^\s+/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
 const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
@@ -55,11 +62,13 @@ const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_QUOTE_DOUBLE = /^"/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_KEYWORD =
-  /^(?:case|continue|default|die|no|else|elsif|eval|exit|for|foreach|given|goto|if|last|next|redo|require|return|select|switch|unless|until|use|wait|when|while)\b/
-
-const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
-const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
+  /^(?:case|continue|default|die|no|else|elsif|eval|exit|for|foreach|given|goto|if|last|next|push|redo|require|return|select|shift|sub|switch|unless|until|use|wait|when|while)\b/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_VARIABLE_NAME = /^[$a-zA-Z\_@%]+/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>\\\-\?]/
 const RE_NUMERIC = /^\d+/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_QUOTE_SINGLE = /^'/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -96,7 +105,31 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
+          switch (next[0]) {
+            case 'unless':
+            case 'foreach':
+            case 'if':
+            case 'while':
+            case 'next':
+            case 'else':
+            case 'last':
+            case 'elsif':
+              token = TokenType.KeywordControl
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'shift':
+            case 'push':
+            case 'eval':
+            case 'exit':
+            case 'die':
+              token = TokenType.Function
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
           state = State.TopLevelContent
         } else if ((next = part.match(RE_PUNCTUATION))) {
           token = TokenType.Punctuation
@@ -110,6 +143,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
           state = State.InsideString
+        } else if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideSingleQuoteString
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.TopLevelContent
@@ -132,6 +168,17 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_QUOTE_SINGLE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         throw new Error('no')
     }

package/extensions/builtin.language-basics-python/extension.json

@@ -7,7 +7,8 @@
       "id": "python",
       "extensions": [".py"],
       "configuration": "./languageConfiguration.json",
-      "tokenize": "src/tokenizePython.js"
+      "tokenize": "src/tokenizePython.js",
+      "firstLine": "^#!\\s*/?.*\\bpython[0-9.-]*\\b"
     }
   ]
 }

package/extensions/builtin.language-basics-python/src/tokenizePython.js

@@ -16,6 +16,8 @@ export const State = {
   InsideDoubleQuoteString: 11,
   InsideLineComment: 12,
   InsideSingleQuoteString: 13,
+  InsideTripleDoubleQuoteString: 14,
+  InsideTripleSingleQuoteString: 15,
 }
 
 /**
@@ -37,6 +39,11 @@ export const TokenType = {
   NewLine: 771,
   Keyword: 951,
   VariableName: 952,
+  LanguageConstant: 953,
+  KeywordReturn: 954,
+  KeywordImport: 955,
+  KeywordControl: 956,
+  Function: 957,
 }
 
 export const TokenMap = {
@@ -55,54 +62,44 @@ export const TokenMap = {
   [TokenType.NewLine]: 'NewLine',
   [TokenType.Keyword]: 'Keyword',
   [TokenType.VariableName]: 'VariableName',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
+  [TokenType.KeywordImport]: 'KeywordImport',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.Function]: 'Function',
 }
 
 const RE_WHITESPACE = /^\s+/
 const RE_DOUBLE_QUOTE = /^"/
+const RE_TRIPLE_DOUBLE_QUOTE = /^"{3}/
 const RE_SINGLE_QUOTE = /^'/
-const RE_ANGLE_BRACKET_OPEN = /^<(?!\s)/
-const RE_ANGLE_BRACKET_CLOSE = /^>/
-const RE_TAGNAME = /^[!\w]+/
 const RE_TEXT = /^.+/
 const RE_ANY_TEXT = /^[^\n]+/
-const RE_SLASH = /^\//
-const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
-const RE_EQUAL_SIGN = /^=/
-const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
-const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
-const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
-const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
-const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
-const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
-const RE_WORD = /^[^\s]+/
-const RE_ANGLE_BRACKET_ONLY = /^</
-const RE_NEWLINE = /^\n/
-const RE_EXCLAMATION_MARK = /^!/
-const RE_SELF_CLOSING = /^\/>/
-const RE_DASH_DASH = /^\-\-/
-const RE_TAG_TEXT = /^[^\s>]+/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"\\]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^'\\]+/
+const RE_STRING_TRIPLE_DOUBLE_QUOTE_CONTENT = /^.+(?=""")/
 const RE_LINE_COMMENT = /^#/
-const RE_NEWLINE_WHITESPACE = /^\n\s*/
-const RE_BLOCK_COMMENT_START = /^\/\*/
-const RE_BLOCK_COMMENT_CONTENT = /^.+(?=\*\/)/s
-const RE_BLOCK_COMMENT_END = /^\*\//
-const RE_UNKNOWN_VALUE = /^[^\}\{\s,"]+/
 const RE_KEYWORD =
   /^(?:__debug__|Ellipsis|yield|with|while|try|true|True|return|raise|pass|or|NotImplemented|not|nonlocal|None|lambda|is|in|import|if|global|from|for|finally|False|false|except|else|elif|del|def|continue|class|break|await|async|assert|as|and)\b/
-const RE_FUNCTION_NAME = /^[a-zA-Z\d]+/
-const RE_ROUND_OPEN = /^\(/
 const RE_VARIABLE_NAME = /^[_a-zA-Z][a-zA-Z\d_]*/
-const RE_ROUND_CLOSE = /^\)/
-const RE_OPERATOR = /^(\+|\-|\*|\/)/
 const RE_NUMERIC =
   /^((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)\b/
 const RE_PUNCTUATION = /^[\(\)\{\}:,\+\-\*&=\/\\\[\]!\.<>%]+/
+const RE_FUNCTION_CALL_NAME = /^[\w]+(?=\s*\()/
+const RE_DECORATOR = /^@[\w]+/
+const RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES = /.*(?=""")/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES = /.*(?=''')/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON = /.*/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_BACKSLASH = /^\\/
+const RE_TRIPLE_SINGLE_QUOTE = /^'{3}/
+const RE_NUMERIC_OCTAL = /^0(?:o|O)?[0-7][0-7_]*(n)?\b/
 
 export const initialLineState = {
   state: State.TopLevelContent,
 }
 
-// TODO maybe reverse order of parameters -> line, context (context may be optional)
+export const hasArrayReturn = true
 
 /**
  *
@@ -124,7 +121,44 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
+          switch (next[0]) {
+            case 'True':
+            case 'False':
+            case 'None':
+              token = TokenType.LanguageConstant
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'from':
+            case 'import':
+              token = TokenType.KeywordImport
+              break
+            case 'if':
+            case 'elif':
+            case 'else':
+            case 'try':
+            case 'catch':
+            case 'finally':
+            case 'while':
+            case 'for':
+            case 'continue':
+            case 'not':
+            case 'except':
+            case 'with':
+            case 'as':
+            case 'raise':
+            case 'break':
+            case 'pass':
+              token = TokenType.KeywordControl
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_FUNCTION_CALL_NAME))) {
+          token = TokenType.Function
           state = State.TopLevelContent
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
           token = TokenType.VariableName
@@ -135,6 +169,15 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC_OCTAL))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleDoubleQuoteString
+        } else if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.InsideTripleSingleQuoteString
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.InsideDoubleQuoteString
@@ -144,6 +187,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_LINE_COMMENT))) {
           token = TokenType.Comment
           state = State.InsideLineComment
+        } else if ((next = part.match(RE_DECORATOR))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_TEXT))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -159,6 +205,12 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
         } else {
           throw new Error('no')
         }
@@ -170,6 +222,49 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_SINGLE_QUOTE))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_BACKSLASH))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleDoubleQuoteString:
+        if ((next = part.match(RE_TRIPLE_DOUBLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_DOUBLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideTripleSingleQuoteString:
+        if ((next = part.match(RE_TRIPLE_SINGLE_QUOTE))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_SINGLE_QUOTES))
+        ) {
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
+        } else if (
+          (next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT_COMMON))
+        ) {
+          next
+          token = TokenType.String
+          state = State.InsideTripleSingleQuoteString
         } else {
           throw new Error('no')
         }
@@ -186,11 +281,9 @@ export const tokenizeLine = (line, lineState) => {
         state
         throw new Error('no')
     }
-    index += next[0].length
-    tokens.push({
-      type: token,
-      length: next[0].length,
-    })
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
   }
   return {
     state,
@@ -198,4 +291,4 @@ export const tokenizeLine = (line, lineState) => {
   }
 }
 
-// tokenizeLine(initialContext, `def add(a,b):`) //?
+tokenizeLine(`'''test'''`, initialLineState) //?