@lvce-editor/shared-process 0.12.0 → 0.13.0

package/extensions/builtin.language-basics-julia/src/tokenizeJulia.js

@@ -22,6 +22,12 @@ export const TokenType = {
   String: 4,
   Numeric: 5,
   Attribute: 6,
+  KeywordReturn: 8,
+  LanguageConstant: 9,
+  KeywordImport: 10,
+  KeywordControl: 11,
+  KeywordOperator: 12,
+  KeywordFunction: 13,
 }
 
 export const TokenMap = {
@@ -34,6 +40,12 @@ export const TokenMap = {
   [TokenType.String]: 'String',
   [TokenType.Numeric]: 'Numeric',
   [TokenType.Attribute]: 'Attribute',
+  [TokenType.KeywordReturn]: 'KeywordReturn',
+  [TokenType.KeywordImport]: 'KeywordImport',
+  [TokenType.LanguageConstant]: 'LanguageConstant',
+  [TokenType.KeywordControl]: 'KeywordControl',
+  [TokenType.KeywordOperator]: 'KeywordOperator',
+  [TokenType.KeywordFunction]: 'KeywordFunction',
 }
 
 export const initialLineState = {
@@ -69,7 +81,46 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Whitespace
           state = State.TopLevelContent
         } else if ((next = part.match(RE_KEYWORD))) {
-          token = TokenType.Keyword
+          switch (next[0]) {
+            case 'true':
+            case 'false':
+              token = TokenType.LanguageConstant
+              break
+            case 'import':
+            case 'export':
+              token = TokenType.KeywordImport
+              break
+            case 'begin':
+            case 'break':
+            case 'catch':
+            case 'continue':
+            case 'do':
+            case 'else':
+            case 'elseif':
+            case 'end':
+            case 'finally':
+            case 'for':
+            case 'if':
+            case 'try':
+            case 'catch':
+            case 'finally':
+            case 'continue':
+            case 'while':
+              token = TokenType.KeywordControl
+              break
+            case 'return':
+              token = TokenType.KeywordReturn
+              break
+            case 'in':
+              token = TokenType.KeywordOperator
+              break
+            case 'function':
+              token = TokenType.KeywordFunction
+              break
+            default:
+              token = TokenType.Keyword
+              break
+          }
           state = State.TopLevelContent
         } else if ((next = part.match(RE_VARIABLE_NAME))) {
           token = TokenType.VariableName

package/extensions/builtin.language-basics-markdown/src/tokenizeMarkdown.js

@@ -41,6 +41,8 @@ export const TokenType = {
   NewLine: 891,
   Heading: 13,
   MarkdownString: 18,
+  MarkdownLinkName: 19,
+  MarkdownLinkUrl: 20,
 }
 
 export const TokenMap = {
@@ -58,6 +60,8 @@ export const TokenMap = {
   [TokenType.PunctuationString]: 'PunctuationString',
   [TokenType.Heading]: 'Heading',
   [TokenType.MarkdownString]: 'MarkdownString',
+  [TokenType.MarkdownLinkName]: 'MarkdownLinkName',
+  [TokenType.MarkdownLinkUrl]: 'MarkdownLinkUrl',
 }
 
 const RE_ANGLE_BRACKET_CLOSE = /^>/
@@ -89,16 +93,47 @@ const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
 const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
 const RE_TAG_TEXT = /^[^\s>]+/
 const RE_TAGNAME = /^[!\w]+/
-const RE_TEXT = /^[^<>\n\`]+/
+const RE_TEXT = /^[^<>\n\`\[\]]+/
 const RE_WHITESPACE = /^\s+/
 const RE_WORD = /^[^\s]+/
 const RE_HEADING = /^\#.*/s
 const RE_TRIPLE_BACKTICK = /^`{3,}/
-const RE_TRIPLE_QUOTED_STRING_CONTENT = /[^`]/s
+const RE_TRIPLE_QUOTED_STRING_CONTENT = /[^`]+/s
 const RE_STRING_ESCAPE = /^\\./
+const RE_LINK = /^\[([^\[\]\\]+?)\]\((.*)\)/
+const RE_ANYTHING_UNTIL_END = /^.+/s
+const RE_ATTRIBUTE_VALUE_UNQUOTED = /^[^<>\s]+/
+
+/**
+ *
+ * @param {string} attribute
+ * @returns {string}
+ */
+const getEmbeddedLanguageId = (attribute) => {
+  switch (attribute) {
+    case 'js':
+      return 'javascript'
+    case 'shell':
+      return 'shellscript'
+    case 'md':
+      return 'markdown'
+    case 'text':
+      return 'plaintext'
+    default:
+      if (attribute.includes(':')) {
+        const parts = attribute.split(':')
+        const firstPart = parts[0]
+        return getEmbeddedLanguageId(firstPart)
+      }
+      return attribute
+  }
+}
 
 export const initialLineState = {
   state: State.TopLevelContent,
+  embeddedLanguage: '',
+  embeddedLanguageStart: 0,
+  embeddedLanguageEnd: 0,
 }
 
 /**
@@ -125,6 +160,9 @@ export const tokenizeLine = (line, lineState) => {
   let tokens = []
   let token = TokenType.None
   let state = lineState.state
+  let embeddedLanguage = lineState.embeddedLanguage
+  let embeddedLanguageStart = lineState.embeddedLanguageStart
+  let embeddedLanguageEnd = lineState.embeddedLanguageEnd
   while (index < line.length) {
     const part = line.slice(index)
     switch (state) {
@@ -152,6 +190,30 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN))) {
           token = TokenType.Text
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_LINK))) {
+          const tokenLength = next[0].length
+          index += tokenLength
+          const linkName = next[1]
+          const linkUrl = next[2]
+          tokens.push(
+            TokenType.Punctuation,
+            1,
+            TokenType.MarkdownLinkName,
+            linkName.length,
+            TokenType.Punctuation,
+            1,
+            TokenType.Punctuation,
+            1,
+            TokenType.MarkdownLinkUrl,
+            linkUrl.length,
+            TokenType.Punctuation,
+            1
+          )
+          tokens.push(token, tokenLength)
+          continue
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
         } else {
           part //?
           console.log({ part })
@@ -273,6 +335,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
           token = TokenType.PunctuationTag
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_ATTRIBUTE_VALUE_UNQUOTED))) {
+          token = TokenType.String
+          state = State.InsideOpeningTag
         } else {
           throw new Error('no')
         }
@@ -314,6 +379,8 @@ export const tokenizeLine = (line, lineState) => {
         if ((next = part.match(RE_WORD))) {
           token = TokenType.String
           state = State.AfterTripleBackTickAfterLanguageId
+          embeddedLanguage = getEmbeddedLanguageId(next[0])
+          embeddedLanguageStart = index + next[0].length
         } else if ((next = part.match(RE_WHITESPACE))) {
           token = TokenType.Whitespace
           state = State.AfterTripleBackTick
@@ -325,10 +392,20 @@ export const tokenizeLine = (line, lineState) => {
         if ((next = part.match(RE_TRIPLE_BACKTICK))) {
           token = TokenType.PunctuationString
           state = State.TopLevelContent
+          embeddedLanguage = ''
+          embeddedLanguageStart = 0
+          embeddedLanguageEnd = 0
         } else if ((next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT))) {
           token = TokenType.String
           state = State.AfterTripleBackTickAfterLanguageId
+          embeddedLanguageStart = Math.min(embeddedLanguageStart, index)
+          embeddedLanguageEnd = index + next[0].length
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Text
+          state = State.AfterTripleBackTickAfterLanguageId
+          embeddedLanguageEnd = index + next[0].length
         } else {
+          part
           throw new Error('no')
         }
         break
@@ -357,5 +434,8 @@ export const tokenizeLine = (line, lineState) => {
   return {
     state,
     tokens,
+    embeddedLanguage,
+    embeddedLanguageStart,
+    embeddedLanguageEnd,
   }
 }

package/extensions/builtin.language-basics-mdx/README.md

@@ -0,0 +1,14 @@
+# Language Basics Mdx
+
+## Contributing
+
+```sh
+git clone git@github.com:lvce-editor/language-basics-mdx.git &&
+cd language-basics-mdx &&
+npm ci &&
+npm test
+```
+
+## Gitpod
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/lvce-editor/language-basics-mdx)

package/extensions/builtin.language-basics-mdx/extension.json

@@ -0,0 +1,12 @@
+{
+  "id": "builtin.language-basics-mdx",
+  "name": "Language Basics Mdx",
+  "description": "Provides syntax highlighting and bracket matching in Mdx files.",
+  "languages": [
+    {
+      "id": "mdx",
+      "extensions": [".mdx"],
+      "tokenize": "src/tokenizeMdx.js"
+    }
+  ]
+}

package/extensions/builtin.language-basics-mdx/src/tokenizeMdx.js

@@ -0,0 +1,395 @@
+/**
+ * @enum number
+ */
+const State = {
+  TopLevelContent: 1,
+  AfterOpeningAngleBracket: 2,
+  InsideOpeningTag: 3,
+  InsideOpeningTagAndHasSeenWhitespace: 4,
+  AfterClosingTagAngleBrackets: 5,
+  AfterClosingTagName: 6,
+  AfterAttributeName: 7,
+  AfterAttributeEqualSign: 8,
+  InsideAttributeDoubleQuote: 9,
+  InsideBlockComment: 10,
+  None: 11,
+  InsideDoubleQuoteString: 12,
+  InsideSingleQuoteString: 13,
+  AfterTripleBackTick: 14,
+  AfterTripleBackTickAfterLanguageId: 15,
+  InsideBackTickString: 16,
+}
+
+export const StateMap = {}
+
+/**
+ * @enum number
+ */
+export const TokenType = {
+  None: 99999999,
+  Numeric: 30,
+  String: 50,
+  Whitespace: 0,
+  Comment: 60,
+  Text: 117,
+  PunctuationTag: 228,
+  TagName: 118,
+  AttributeName: 119,
+  Punctuation: 10,
+  Error: 141,
+  PunctuationString: 11,
+  NewLine: 891,
+  Heading: 13,
+  MarkdownString: 18,
+  MarkdownLinkName: 19,
+  MarkdownLinkUrl: 20,
+}
+
+export const TokenMap = {
+  [TokenType.None]: 'None',
+  [TokenType.Numeric]: 'Numeric',
+  [TokenType.String]: 'String',
+  [TokenType.Whitespace]: 'Whitespace',
+  [TokenType.Comment]: 'Comment',
+  [TokenType.Text]: 'Text',
+  [TokenType.PunctuationTag]: 'PunctuationTag',
+  [TokenType.TagName]: 'TagName',
+  [TokenType.AttributeName]: 'AttributeName',
+  [TokenType.Punctuation]: 'Punctuation',
+  [TokenType.Error]: 'Error',
+  [TokenType.PunctuationString]: 'PunctuationString',
+  [TokenType.Heading]: 'Heading',
+  [TokenType.MarkdownString]: 'MarkdownString',
+  [TokenType.MarkdownLinkName]: 'MarkdownLinkName',
+  [TokenType.MarkdownLinkUrl]: 'MarkdownLinkUrl',
+}
+
+const RE_ANGLE_BRACKET_CLOSE = /^>/
+const RE_ANGLE_BRACKET_ONLY = /^</
+const RE_ANGLE_BRACKET_OPEN = /^</
+const RE_ANGLE_BRACKET_OPEN_TAG = /^<(?!\s)/
+const RE_ANY_TEXT = /^[^\n]+/
+const RE_ATTRIBUTE_NAME = /^[a-zA-Z\d\-]+/
+const RE_BLOCK_COMMENT_CONTENT = /^.(?:.*?)(?=-->|$)/s
+const RE_BLOCK_COMMENT_END = /^-->/
+const RE_BLOCK_COMMENT_START = /^<!--/
+const RE_DASH_DASH = /^\-\-/
+const RE_DOUBLE_QUOTE = /^"/
+const RE_EQUAL_SIGN = /^=/
+const RE_EXCLAMATION_MARK = /^!/
+const RE_INVALID_INSIDE_ClOSING_TAG = /^[^a-zA-Z>]/
+const RE_INVALID_INSIDE_OPENING_TAG = /^[^a-zA-Z>]/
+const RE_NEWLINE = /^\n/
+const RE_NEWLINE_WHITESPACE = /^\n\s*/
+const RE_NOT_TAGNAME = /^[^a-zA-Z\d]+/
+const RE_PUNCTUATION = /^[<;'".,]/
+const RE_PUNCTUATION_SELF_CLOSING = /^\/>/
+const RE_SELF_CLOSING = /^\/>/
+const RE_SINGLE_QUOTE = /^'/
+const RE_QUOTE_BACKTICK = /^`/
+const RE_SLASH = /^\//
+const RE_STRING_BACKTICK_QUOTE_CONTENT = /^[^`]+/
+const RE_STRING_DOUBLE_QUOTE_CONTENT = /^[^"]+/
+const RE_STRING_SINGLE_QUOTE_CONTENT = /^[^']+/
+const RE_TAG_TEXT = /^[^\s>]+/
+const RE_TAGNAME = /^[!\w]+/
+const RE_TEXT = /^[^<>\n\`\[\]]+/
+const RE_WHITESPACE = /^\s+/
+const RE_WORD = /^[^\s]+/
+const RE_HEADING = /^\#.*/s
+const RE_TRIPLE_BACKTICK = /^`{3,}/
+const RE_TRIPLE_QUOTED_STRING_CONTENT = /[^`]/s
+const RE_STRING_ESCAPE = /^\\./
+const RE_LINK = /^\[([^\[\]\\]+?)\]\((.*)\)/
+const RE_ANYTHING_UNTIL_END = /^.+/s
+
+export const initialLineState = {
+  state: State.TopLevelContent,
+}
+
+/**
+ *
+ * @param {any} lineStateA
+ * @param {any} lineStateB
+ * @returns
+ */
+export const isLineStateEqual = (lineStateA, lineStateB) => {
+  return lineStateA.state === lineStateB.state
+}
+
+export const hasArrayReturn = true
+
+/**
+ *
+ * @param {string} line
+ * @param {any} lineState
+ * @returns
+ */
+export const tokenizeLine = (line, lineState) => {
+  let next = null
+  let index = 0
+  let tokens = []
+  let token = TokenType.None
+  let state = lineState.state
+  while (index < line.length) {
+    const part = line.slice(index)
+    switch (state) {
+      case State.TopLevelContent:
+        if ((next = part.match(RE_BLOCK_COMMENT_START))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN_TAG))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterOpeningAngleBracket
+        } else if ((next = part.match(RE_HEADING))) {
+          token = TokenType.Heading
+        } else if ((next = part.match(RE_TRIPLE_BACKTICK))) {
+          token = TokenType.PunctuationString
+          state = State.AfterTripleBackTick
+        } else if ((next = part.match(RE_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.InsideBackTickString
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANGLE_BRACKET_OPEN))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_LINK))) {
+          const tokenLength = next[0].length
+          index += tokenLength
+          const linkName = next[1]
+          const linkUrl = next[2]
+          tokens.push(
+            TokenType.Punctuation,
+            1,
+            TokenType.MarkdownLinkName,
+            linkName.length,
+            TokenType.Punctuation,
+            1,
+            TokenType.Punctuation,
+            1,
+            TokenType.MarkdownLinkUrl,
+            linkUrl.length,
+            TokenType.Punctuation,
+            1
+          )
+          tokens.push(token, tokenLength)
+          continue
+        } else if ((next = part.match(RE_ANYTHING_UNTIL_END))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          console.log({ part })
+          throw new Error('no')
+        }
+        break
+      case State.AfterOpeningAngleBracket:
+        if ((next = part.match(RE_TAGNAME))) {
+          token = TokenType.TagName
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_SLASH))) {
+          token = TokenType.PunctuationTag
+          state = State.AfterClosingTagAngleBrackets
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANY_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.InsideOpeningTag:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EXCLAMATION_MARK))) {
+          token = TokenType.PunctuationTag
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.InsideOpeningTagAndHasSeenWhitespace
+        } else if ((next = part.match(RE_DASH_DASH))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.InsideOpeningTag
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterClosingTagAngleBrackets:
+        if ((next = part.match(RE_TAGNAME))) {
+          token = TokenType.TagName
+          state = State.AfterClosingTagName
+        } else if ((next = part.match(RE_ANY_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterClosingTagName:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideOpeningTagAndHasSeenWhitespace:
+        if ((next = part.match(RE_ATTRIBUTE_NAME))) {
+          token = TokenType.AttributeName
+          state = State.AfterAttributeName
+        } else if ((next = part.match(RE_SELF_CLOSING))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.TopLevelContent
+        } else {
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterAttributeName:
+        if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_EQUAL_SIGN))) {
+          token = TokenType.Punctuation
+          state = State.AfterAttributeEqualSign
+        } else if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_TAG_TEXT))) {
+          token = TokenType.Text
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.InsideOpeningTagAndHasSeenWhitespace
+        } else {
+          // TODO check quotation mark
+          part //?
+          throw new Error('no')
+        }
+        break
+      case State.AfterAttributeEqualSign:
+        if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideDoubleQuoteString
+        } else if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideSingleQuoteString
+        } else if ((next = part.match(RE_ANGLE_BRACKET_CLOSE))) {
+          token = TokenType.PunctuationTag
+          state = State.TopLevelContent
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideDoubleQuoteString:
+        if ((next = part.match(RE_DOUBLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_STRING_DOUBLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideDoubleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideSingleQuoteString:
+        if ((next = part.match(RE_SINGLE_QUOTE))) {
+          token = TokenType.PunctuationString
+          state = State.InsideOpeningTag
+        } else if ((next = part.match(RE_STRING_SINGLE_QUOTE_CONTENT))) {
+          token = TokenType.String
+          state = State.InsideSingleQuoteString
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.InsideBlockComment:
+        if ((next = part.match(RE_BLOCK_COMMENT_END))) {
+          token = TokenType.Comment
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_BLOCK_COMMENT_CONTENT))) {
+          token = TokenType.Comment
+          state = State.InsideBlockComment
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterTripleBackTick:
+        if ((next = part.match(RE_WORD))) {
+          token = TokenType.String
+          state = State.AfterTripleBackTickAfterLanguageId
+        } else if ((next = part.match(RE_WHITESPACE))) {
+          token = TokenType.Whitespace
+          state = State.AfterTripleBackTick
+        } else {
+          throw new Error('no')
+        }
+        break
+      case State.AfterTripleBackTickAfterLanguageId:
+        if ((next = part.match(RE_TRIPLE_BACKTICK))) {
+          token = TokenType.PunctuationString
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_TRIPLE_QUOTED_STRING_CONTENT))) {
+          token = TokenType.String
+          state = State.AfterTripleBackTickAfterLanguageId
+        } else if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Text
+          state = State.AfterTripleBackTickAfterLanguageId
+        } else {
+          part
+          throw new Error('no')
+        }
+        break
+      case State.InsideBackTickString:
+        if ((next = part.match(RE_QUOTE_BACKTICK))) {
+          token = TokenType.Punctuation
+          state = State.TopLevelContent
+        } else if ((next = part.match(RE_STRING_BACKTICK_QUOTE_CONTENT))) {
+          token = TokenType.MarkdownString
+          state = State.InsideBackTickString
+        } else if ((next = part.match(RE_STRING_ESCAPE))) {
+          token = TokenType.String
+          state = State.InsideBackTickString
+        } else {
+          throw new Error('no')
+        }
+        break
+      default:
+        state
+        throw new Error('no')
+    }
+    const tokenLength = next[0].length
+    index += tokenLength
+    tokens.push(token, tokenLength)
+  }
+  return {
+    state,
+    tokens,
+  }
+}