@lvce-editor/shared-process 0.4.4 → 0.6.0

package/extensions/builtin.language-basics-rust/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-rust",
+  "name": "Language Basics Rust",
+  "description": "Provides syntax highlighting and bracket matching in Rust files.",
+  "languages": [
+    {
+      "id": "rust",
+      "extensions": [".rs"],
+      "tokenize": "src/tokenizeRust.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-rust/src/tokenizeRust.js

@@ -0,0 +1,158 @@
+/**
+ * @enum number
+ */
+export const State = {
+  TopLevelContent: 1,
+  InsideString: 2,
+  InsideLineComment: 3,
+}
+
+export const StateMap = {
+  [State.TopLevelContent]: 'TopLevelContent',
+}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  None: 1,
+  Whitespace: 2,
+  PunctuationString: 3,
+  String: 4,
+  Keyword: 5,
+  Numeric: 6,
+  Punctuation: 7,
+  VariableName: 8,
+  Comment: 885,
+  Text: 9,
+  LanguageConstant: 10,
+}
+
+export const TokenMap = {
+  [TokenType.None]: 'None',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.String]: 'String',
+  [TokenType.Keyword]: 'Keyword',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.VariableName]: 'VariableName',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Text]: 'Text',
+
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+}
+
+const RE_LINE_COMMENT_START = /^\/\//
+const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
+const RE_WHITESPACE = /^ +/
+const RE_CURLY_OPEN = /^\{/
+const RE_CURLY_CLOSE = /^\}/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_COLON = /^:/
+const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_SEMICOLON = /^;/
+const RE_COMMA = /^,/
+const RE_ANYTHING = /^.*/s
+const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
+const RE_QUOTE_DOUBLE = /^"/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_KEYWORD =
+  /^(?:as|break|const|continue|crate|else|enum|extern|false|fn|for|if|impl|in|let|loop|match|mod|move|mut|pub|ref|return|self|static|struct|super|trait|true|type|unsafe|where|while)\b/
+
+const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
+const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
+const RE_NUMERIC = /^\d+/
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+  tokens: [],
+}
+
+export const hasArrayReturn = true
+
+/**
+ * @param {string} line
+ * @param {any} lineState
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_KEYWORD))) {
+          switch (next[0]) {
+            case 'true':
+            case 'false':
+              token = TokenType.LanguageConstant
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_PUNCTUATION))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_VARIABLE_NAME))) {
+          token = TokenType.VariableName
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideString
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideString:
+        if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        throw new Error('no')
+    }
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
+  }
+  if (state === State.InsideLineComment) {
+    state = State.TopLevelContent
+  }
+  return {
+    state,
+    tokens,
+  }
+}

package/extensions/builtin.language-basics-shellscript/src/tokenizeShellScript.js

@@ -4,6 +4,7 @@
 export const State = {
   TopLevelContent: 1,
   InsideString: 2,
+  InsideLineComment: 3,
 }
 
 export const StateMap = {
@@ -22,6 +23,8 @@ export const TokenType = {
   Numeric: 6,
   Punctuation: 7,
   VariableName: 8,
+  Comment: 885,
+  Text: 9,
 }
 
 export const TokenMap = {
@@ -33,8 +36,11 @@ export const TokenMap = {
   [TokenType.Numeric]: 'Numeric',
   [TokenType.Punctuation]: 'Punctuation',
   [TokenType.VariableName]: 'VariableName',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Text]: 'Text',
 }
 
+const RE_LINE_COMMENT_START = /^#/
 const RE_SELECTOR = /^[\.a-zA-Z\d\-\:>]+/
 const RE_WHITESPACE = /^ +/
 const RE_CURLY_OPEN = /^\{/
@@ -44,14 +50,14 @@ const RE_COLON = /^:/
 const RE_PROPERTY_VALUE = /^[^;\}]+/
 const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
-const RE_ANYTHING = /^.*/
+const RE_ANYTHING = /^.*/s
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_QUOTE_DOUBLE = /^"/
 const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_KEYWORD =
   /^(?:alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|popd|printf|pushd|pwd|read|readonly|set|shift|shopt|source|suspend|test|times|trap|true|type|ulimit|umask|unalias|unset|wait)\b/
 
-const RE_VARIABLE_NAME = /^[a-zA-Z]+/
+const RE_VARIABLE_NAME = /^[a-zA-Z\_]+/
 const RE_PUNCTUATION = /^[:,;\{\}\[\]\.=\(\)<>]/
 const RE_NUMERIC = /^\d+/
 
@@ -60,12 +66,21 @@ export const initialLineState = {
   tokens: [],
 }
 
+/**
+ *
+ * @param {any} lineStateA
+ * @param {any} lineStateB
+ * @returns
+ */
 export const isEqualLineState = (lineStateA, lineStateB) => {
   return lineStateA.state === lineStateB.state
 }
 
+export const hasArrayReturn = true
+
 /**
  * @param {string} line
+ * @param {any} lineState
  */
 export const tokenizeLine = (line, lineState) => {
   let next = null
@@ -95,6 +110,12 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
           token = TokenType.PunctuationString
           state = State.InsideString
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else {
           part //?
           throw new Error('no')
@@ -111,14 +132,23 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.InsideLineComment:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         throw new Error('no')
     }
-    index += next[0].length
-    tokens.push({
-      type: token,
-      length: next[0].length,
-    })
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
+  }
+  if (state === State.InsideLineComment) {
+    state = State.TopLevelContent
   }
   return {
     state,

package/extensions/builtin.language-basics-toml/extension.json

@@ -11,7 +11,7 @@
         ".tml",
         ".replit"
       ],
-      "filenames": [
+      "fileNames": [
         "Cargo.lock",
         "Gopkg.lock"
       ],

package/extensions/builtin.language-basics-toml/src/tokenizeToml.js

@@ -4,6 +4,9 @@
 export const State = {
   TopLevelContent: 1,
   InsideLineComment: 2,
+  AfterPropertyName: 3,
+  AfterPropertyNameAfterEqualSign: 4,
+  InsideString: 5,
 }
 
 export const StateMap = {
@@ -17,12 +20,19 @@ export const StateMap = {
 export const TokenType = {
   CssSelector: 1,
   Whitespace: 2,
+  PunctuationString: 3,
+  String: 4,
   None: 57,
   Unknown: 881,
   NewLine: 884,
   Comment: 885,
   Query: 886,
   Text: 887,
+  Numeric: 15,
+  PropertyName: 12,
+  PropertyValueString: 14,
+  Punctuation: 13,
+  LanguageConstant: 11,
 }
 
 export const TokenMap = {
@@ -34,19 +44,29 @@ export const TokenMap = {
   [TokenType.Comment]: 'Comment',
   [TokenType.Query]: 'Query',
   [TokenType.Text]: 'Text',
+  [TokenType.PropertyName]: 'JsonPropertyName',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.PropertyValueString]: 'PropertyValueString',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.String]: 'String',
 }
 
 const RE_LINE_COMMENT_START = /^#/
 const RE_WHITESPACE = /^ +/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
-const RE_PROPERTY_NAME = /^[a-zA-Z\-]+\b/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-\_\d]+\b(?=\s*=)/
 const RE_COLON = /^:/
 const RE_PROPERTY_VALUE = /^[^;\}]+/
 const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
 const RE_ANYTHING = /^.*/
-const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_NUMERIC_INTEGER = /^((?:[\+\-]?(0|([1-9](([0-9]|_[0-9])+)?))))/
+const RE_NUMERIC_FLOAT = /^([\+\-]?(0|([1-9](([0-9]|_[0-9])+)?))(?:(?:\.(0|([1-9](([0-9]|_[0-9])+)?)))?[eE][\+\-]?[1-9]_?[0-9]*|(?:\.[0-9_]*)))/
+const RE_NAN = /^(?:[\+\-]?nan)/
+const RE_INF = /^(?:[\+\-]?inf)/
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_BLOCK_COMMENT_START = /^\/\*/
 const RE_BLOCK_COMMENT_END = /^\*\//
@@ -62,6 +82,11 @@ const RE_STAR = /^\*/
 const RE_QUERY_NAME = /^[a-z\-]+/
 const RE_QUERY_CONTENT = /^[^\)]+/
 const RE_COMBINATOR = /^[\+\>\~]/
+const RE_LANGUAGE_CONSTANT = /^(?:true|false)\b/
+const RE_EQUAL_SIGN = /^=/
+const RE_LOCAL_TIME = /\d{2}:\d{2}:\d{2}(?:\.\d+)?/
+const RE_QUOTE_DOUBLE = /^"/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 
 export const initialLineState = {
   state: State.TopLevelContent,
@@ -69,6 +94,8 @@ export const initialLineState = {
   stack: [],
 }
 
+export const hasArrayReturn = true
+
 /**
  * @param {string} line
  * @param {any} lineState
@@ -83,7 +110,10 @@ export const tokenizeLine = (line, lineState) => {
     const part = line.slice(index)
     switch (state) {
       case State.TopLevelContent:
-        if ((next = part.match(RE_LINE_COMMENT_START))) {
+        if ((next = part.match(RE_PROPERTY_NAME))) {
+          token = TokenType.PropertyName
+          state = State.AfterPropertyName
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
           token = TokenType.Comment
           state = State.InsideLineComment
         } else if ((next = part.match(RE_WHITESPACE))) {
@@ -105,15 +135,70 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.AfterPropertyName:
+        if ((next = part.match(RE_EQUAL_SIGN))) {
+          token = TokenType.Punctuation
+          state = State.AfterPropertyNameAfterEqualSign
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyName
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterPropertyNameAfterEqualSign:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyNameAfterEqualSign
+        } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
+          token = TokenType.LanguageConstant
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LOCAL_TIME))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC_INTEGER))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_NUMERIC_FLOAT))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_NAN))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_INF))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.Punctuation
+          state = State.InsideString
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.PropertyValueString
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideString:
+        if ((next = part.match(RE_QUOTE_DOUBLE))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideString
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         console.log({ state, line })
         throw new Error('no')
     }
-    index += next[0].length
-    tokens.push({
-      type: token,
-      length: next[0].length,
-    })
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
   }
   return {
     state,

package/extensions/builtin.language-basics-yaml/src/tokenizeYaml.js

@@ -7,6 +7,11 @@ export const State = {
   AfterPropertyName: 3,
   AfterPropertyNameAfterColon: 4,
   AfterDash: 5,
+  AfterPropertyNameAfterColonAfterNewLine: 6,
+  AfterPipe: 7,
+  InsideMultiLineString: 8,
+  InsideMultiLineStringAfterWhitespace: 9,
+  AfterPropertyValue: 10,
 }
 
 export const StateMap = {
@@ -31,6 +36,7 @@ export const TokenType = {
   Text: 887,
   Unknown: 881,
   Whitespace: 2,
+  String: 188,
 }
 
 export const TokenMap = {
@@ -47,18 +53,20 @@ export const TokenMap = {
   [TokenType.Whitespace]: 'Whitespace',
   [TokenType.Punctuation]: 'Punctuation',
   [TokenType.PropertyValueString]: 'String',
+  [TokenType.String]: 'String',
 }
 
 const RE_LINE_COMMENT_START = /^#/
 const RE_WHITESPACE = /^ +/
 const RE_CURLY_OPEN = /^\{/
 const RE_CURLY_CLOSE = /^\}/
-const RE_PROPERTY_NAME = /^[a-zA-Z\-\_\d\s]+\b(?=\s*:)/
-const RE_PROPERTY_VALUE = /^[^;\}]+/
+const RE_PROPERTY_NAME = /^[a-zA-Z\-\_\d\s]+\b(?=\s*:(\s+|$))/
+const RE_PROPERTY_VALUE_1 = /^.+(?=\s+#)/s
 const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
 const RE_ANYTHING = /^.*/
-const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))/
+const RE_ANYTHING_BUT_COMMENT = /^[^#]+/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))(?=\s|$)/
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_BLOCK_COMMENT_START = /^\/\*/
 const RE_BLOCK_COMMENT_END = /^\*\//
@@ -74,18 +82,35 @@ const RE_STAR = /^\*/
 const RE_QUERY_NAME = /^[a-z\-]+/
 const RE_QUERY_CONTENT = /^[^\)]+/
 const RE_COMBINATOR = /^[\+\>\~]/
-const RE_LANGUAGE_CONSTANT = /^(?:true|false)/
+const RE_LANGUAGE_CONSTANT = /^(?:true|false|null)(?!#)/
 const RE_COLON = /^:/
 const RE_DASH = /^\-/
+const RE_WORDS = /^[\w\s]*\w/
+const RE_MULTI_LINE_STRING_START = /^(?:\||>\-|>)/
+const RE_KEY_PRE = /^\s*(\-\s*)?/
 
 export const initialLineState = {
   state: State.TopLevelContent,
   tokens: [],
-  stack: [],
+  indent: '',
+  keyOffset: 0,
 }
 
 export const hasArrayReturn = true
 
+/**
+ *
+ * @param {string} line
+ * @returns {number}
+ */
+const getKeyOffset = (line) => {
+  const whiteSpaceMatch = line.match(RE_KEY_PRE)
+  if (whiteSpaceMatch) {
+    return whiteSpaceMatch[0].length
+  }
+  return 0
+}
+
 /**
  * @param {string} line
  * @param {any} lineState
@@ -96,6 +121,7 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let keyOffset = lineState.keyOffset
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -118,6 +144,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_WORDS))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -155,6 +184,15 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else if ((next = part.match(RE_MULTI_LINE_STRING_START))) {
+          token = TokenType.Punctuation
+          state = State.AfterPipe
+        } else if ((next = part.match(RE_PROPERTY_VALUE_1))) {
+          token = TokenType.PropertyValueString
+          state = State.AfterPropertyValue
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.PropertyValueString
           state = State.TopLevelContent
@@ -162,6 +200,26 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.AfterPropertyValue:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyValue
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
+      case State.AfterPipe:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPipe
+        } else {
+          throw new Error('no')
+        }
+        break
       case State.AfterDash:
         if ((next = part.match(RE_WHITESPACE))) {
           token = TokenType.Whitespace
@@ -175,6 +233,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_MULTI_LINE_STRING_START))) {
+          token = TokenType.Punctuation
+          state = State.AfterPipe
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.PropertyValueString
           state = State.TopLevelContent
@@ -182,19 +243,86 @@ export const tokenizeLine = (line, lineState) => {
           throw new Error('no')
         }
         break
+      case State.AfterPropertyNameAfterColonAfterNewLine:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterPropertyNameAfterColonAfterNewLine
+        } else if ((next = part.match(RE_DASH))) {
+          token = TokenType.Punctuation
+          state = State.AfterDash
+        } else if ((next = part.match(RE_PROPERTY_NAME))) {
+          token = TokenType.PropertyName
+          state = State.AfterPropertyName
+        } else if ((next = part.match(RE_NUMERIC))) {
+          token = TokenType.Numeric
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LANGUAGE_CONSTANT))) {
+          token = TokenType.LanguageConstant
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_WORDS))) {
+          token = TokenType.PropertyValueString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LINE_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideLineComment
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
+      case State.InsideMultiLineString:
+        if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          if (next[0].length > keyOffset) {
+            state = State.InsideMultiLineStringAfterWhitespace
+          } else {
+            state = State.TopLevelContent
+          }
+        } else if ((next = part.match(RE_PROPERTY_NAME))) {
+          token = TokenType.PropertyName
+          state = State.AfterPropertyName
+        } else if ((next = part.match(RE_DASH))) {
+          token = TokenType.Punctuation
+          state = State.AfterDash
+        } else if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideMultiLineStringAfterWhitespace:
+        if ((next = part.match(RE_ANYTHING))) {
+          token = TokenType.String
+          state = State.InsideMultiLineString
+        } else {
+          throw new Error('no')
+        }
+        break
       default:
         console.log({ state, line })
         throw new Error('no')
     }
-    const length = next[0].length
-    index += length
-    tokens.push(token, length)
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
   }
-  if (state === State.AfterPropertyNameAfterColon) {
-    state = State.TopLevelContent
+  switch (state) {
+    case State.AfterPropertyNameAfterColon:
+      state = State.AfterPropertyNameAfterColonAfterNewLine
+      break
+    case State.AfterPipe:
+      keyOffset = getKeyOffset(line)
+      keyOffset
+      line
+      state = State.InsideMultiLineString
+      break
+    default:
+      break
   }
   return {
     state,
     tokens,
+    keyOffset,
   }
 }