@xnoxs/flux-lang 3.1.1

package/src/self/lexer.flux

@@ -0,0 +1,529 @@
+// ============================================================
+//  Flux Self-Hosted Lexer
+//  src/self/lexer.flux — written in Flux, compiled by stage-0
+// ============================================================
+
+export val T = {
+  NUMBER: 'NUMBER', STRING: 'STRING', BOOL: 'BOOL', NULL: 'NULL', IDENT: 'IDENT',
+  VAR: 'VAR', VAL: 'VAL', FN: 'FN', RETURN: 'RETURN',
+  IF: 'IF', ELSE: 'ELSE', FOR: 'FOR', IN: 'IN',
+  WHILE: 'WHILE', BREAK: 'BREAK', CONTINUE: 'CONTINUE', DO: 'DO',
+  CLASS: 'CLASS', EXTENDS: 'EXTENDS', SELF: 'SELF', NEW: 'NEW',
+  INTERFACE: 'INTERFACE', IMPLEMENTS: 'IMPLEMENTS',
+  PRIVATE: 'PRIVATE', PUBLIC: 'PUBLIC', PROTECTED: 'PROTECTED',
+  READONLY: 'READONLY', STATIC: 'STATIC', ABSTRACT: 'ABSTRACT', OVERRIDE: 'OVERRIDE',
+  MATCH: 'MATCH', WHEN: 'WHEN',
+  IMPORT: 'IMPORT', EXPORT: 'EXPORT', FROM: 'FROM', AS: 'AS', DEFAULT: 'DEFAULT',
+  AND: 'AND', OR: 'OR', NOT: 'NOT',
+  ASYNC: 'ASYNC', AWAIT: 'AWAIT',
+  TRY: 'TRY', CATCH: 'CATCH', FINALLY: 'FINALLY', THROW: 'THROW',
+  TYPEOF: 'TYPEOF', INSTANCEOF: 'INSTANCEOF', TYPE: 'TYPE',
+  ENUM: 'ENUM', SATISFIES: 'SATISFIES', IS: 'IS', CONST: 'CONST',
+  PLUS: 'PLUS', MINUS: 'MINUS', STAR: 'STAR', SLASH: 'SLASH', PERCENT: 'PERCENT',
+  REGEX: 'REGEX', STARSTAR: 'STARSTAR',
+  EQ: 'EQ', EQEQ: 'EQEQ', NEQ: 'NEQ', EQEQEQ: 'EQEQEQ', NEQEQ: 'NEQEQ',
+  LT: 'LT', LTE: 'LTE', GT: 'GT', GTE: 'GTE',
+  PLUSEQ: 'PLUSEQ', MINUSEQ: 'MINUSEQ', STAREQ: 'STAREQ', SLASHEQ: 'SLASHEQ',
+  PERCENTEQ: 'PERCENTEQ',
+  PLUSPLUS: 'PLUSPLUS', MINUSMINUS: 'MINUSMINUS',
+  AMPERSAND: 'AMPERSAND', ANDAND: 'ANDAND',
+  PIPEB: 'PIPEB', OROR: 'OROR',
+  CARET: 'CARET', TILDE: 'TILDE', LSHIFT: 'LSHIFT', RSHIFT: 'RSHIFT',
+  ARROW: 'ARROW', FATARROW: 'FATARROW',
+  PIPE: 'PIPE', DOTDOT: 'DOTDOT', DOTDOTDOT: 'DOTDOTDOT',
+  WILDCARD: 'WILDCARD', NULLISH: 'NULLISH',
+  QUESTIONDOT: 'QUESTIONDOT', BANG: 'BANG', AT: 'AT',
+  LPAREN: 'LPAREN', RPAREN: 'RPAREN',
+  LBRACKET: 'LBRACKET', RBRACKET: 'RBRACKET',
+  LBRACE: 'LBRACE', RBRACE: 'RBRACE',
+  COMMA: 'COMMA', DOT: 'DOT', COLON: 'COLON', QUESTION: 'QUESTION',
+  NEWLINE: 'NEWLINE', INDENT: 'INDENT', DEDENT: 'DEDENT', EOF: 'EOF',
+}
+
+export val TokenType = T
+
+val KEYWORDS = {
+  "var": 'VAR', "val": 'VAL', "fn": 'FN', "return": 'RETURN',
+  "if": 'IF', "else": 'ELSE', "for": 'FOR', "in": 'IN',
+  "while": 'WHILE', "break": 'BREAK', "continue": 'CONTINUE', "do": 'DO',
+  "class": 'CLASS', "extends": 'EXTENDS', "self": 'SELF', "new": 'NEW',
+  "interface": 'INTERFACE', "implements": 'IMPLEMENTS',
+  "private": 'PRIVATE', "public": 'PUBLIC', "protected": 'PROTECTED',
+  "readonly": 'READONLY', "static": 'STATIC', "abstract": 'ABSTRACT', "override": 'OVERRIDE',
+  "match": 'MATCH', "when": 'WHEN',
+  "import": 'IMPORT', "export": 'EXPORT', "from": 'FROM', "as": 'AS', "default": 'DEFAULT',
+  "and": 'AND', "or": 'OR', "not": 'NOT',
+  "async": 'ASYNC', "await": 'AWAIT',
+  "try": 'TRY', "catch": 'CATCH', "finally": 'FINALLY', "throw": 'THROW',
+  "typeof": 'TYPEOF', "instanceof": 'INSTANCEOF', "type": 'TYPE',
+  "enum": 'ENUM', "satisfies": 'SATISFIES', "is": 'IS', "const": 'CONST',
+  "true": '__TRUE__', "false": '__FALSE__', "null": '__NULL__',
+}
+
+export class Lexer:
+  src: string
+  pos: int
+  line: int
+  col: int
+  tokens: any[]
+  indentStack: any[]
+  nestDepth: int
+
+  fn err(msg):
+    throw new Error("[Lexer {self.line}:{self.col}] {msg}")
+
+  fn ch(n = 0):
+    return self.src[self.pos + n] ?? ''
+
+  fn adv():
+    val c = self.src[self.pos]
+    self.pos = self.pos + 1
+    if c == '\n':
+      self.line = self.line + 1
+      self.col = 1
+    else:
+      self.col = self.col + 1
+    return c
+
+  fn tok(typ, value, l, c):
+    val v = value != undefined ? value : typ
+    val ln = l != undefined ? l : self.line
+    val co = c != undefined ? c : self.col
+    self.tokens.push({ type: typ, value: v, line: ln, col: co })
+
+  fn applyIndent(indent):
+    if self.nestDepth > 0: return
+    val top = self.indentStack[self.indentStack.length - 1]
+    if indent > top:
+      self.indentStack.push(indent)
+      self.tok(T.INDENT, indent, self.line, 1)
+    else if indent < top:
+      while self.indentStack.length > 1 and self.indentStack[self.indentStack.length - 1] > indent:
+        self.indentStack.pop()
+        self.tok(T.DEDENT, null, self.line, 1)
+      if self.indentStack[self.indentStack.length - 1] != indent:
+        self.err("Inconsistent indentation ({indent} spaces)")
+
+  fn scanBacktick(l, c):
+    self.adv()
+    var s = ''
+    while self.pos < self.src.length and self.ch() != '`':
+      if self.ch() == '\\':
+        self.adv()
+        val e = self.adv()
+        if e == 'n': s += '\n'
+        else if e == 't': s += '\t'
+        else if e == '`': s += '`'
+        else if e == '\\': s += '\\'
+        else: s += '\\' + e
+      else:
+        s += self.adv()
+    if self.ch() != '`': self.err('Unterminated backtick string')
+    self.adv()
+    val rawLines = s.split('\n')
+    val trimmed = rawLines.map(ln -> ln.trimEnd())
+    while trimmed.length and not trimmed[0].trim(): trimmed.shift()
+    while trimmed.length and not trimmed[trimmed.length - 1].trim(): trimmed.pop()
+    var minIndent = 999999
+    for ln in trimmed:
+      if ln.trim():
+        val m = ln.match(/^(\s*)/)
+        val ind = m[1].length
+        if ind < minIndent: minIndent = ind
+    if minIndent == 999999: minIndent = 0
+    val dedented = trimmed.map(ln -> ln.slice(minIndent))
+    self.tok(T.STRING, dedented.join('\n'), l, c)
+
+  fn scanStr(l, c):
+    self.adv()
+    val parts = []
+    var text = ''
+    while self.pos < self.src.length and self.ch() != '"':
+      if self.ch() == '\\':
+        self.adv()
+        val e = self.adv()
+        if e == 'n': text += '\n'
+        else if e == 't': text += '\t'
+        else if e == '"': text += '"'
+        else if e == "'": text += "'"
+        else if e == '\\': text += '\\'
+        else if e == '{': text += '{'
+        else if e == '}': text += '}'
+        else: text += '\\' + e
+      else if self.ch() == '{':
+        parts.push({ type: 'text', value: text })
+        text = ''
+        self.adv()
+        var depth = 1
+        var expr = ''
+        while self.pos < self.src.length:
+          val cc = self.ch()
+          if cc == '{': depth = depth + 1
+          if cc == '}':
+            depth = depth - 1
+            if depth == 0: break
+          if cc == '\\' and (self.src[self.pos + 1] == '"' or self.src[self.pos + 1] == "'"):
+            self.adv()
+            expr += self.adv()
+          else:
+            expr += self.adv()
+        if self.ch() != '}': self.err('Unclosed string interpolation')
+        self.adv()
+        parts.push({ type: 'expr', value: expr.trim() })
+      else:
+        text += self.adv()
+    if self.ch() != '"': self.err('Unterminated string')
+    self.adv()
+    parts.push({ type: 'text', value: text })
+    if parts.some(p -> p.type == 'expr'):
+      self.tok(T.STRING, { template: true, parts }, l, c)
+    else:
+      self.tok(T.STRING, parts.map(p -> p.value).join(''), l, c)
+
+  fn scanRegexBody(l, c):
+    var pattern = ''
+    var inClass = false
+    while self.pos < self.src.length:
+      val ch = self.ch()
+      if ch == '\n': self.err('Unterminated regex literal')
+      if ch == '\\':
+        pattern += self.adv()
+        if self.pos < self.src.length: pattern += self.adv()
+        continue
+      if ch == '[':
+        inClass = true
+        pattern += self.adv()
+        continue
+      if ch == ']':
+        inClass = false
+        pattern += self.adv()
+        continue
+      if ch == '/' and not inClass: break
+      pattern += self.adv()
+    if self.ch() != '/': self.err('Unterminated regex literal')
+    self.adv()
+    var flags = ''
+    while /[gimsuy]/.test(self.ch()): flags += self.adv()
+    self.tok(T.REGEX, { pattern, flags }, l, c)
+
+  fn scanBlockComment():
+    self.adv()
+    self.adv()
+    while self.pos < self.src.length:
+      if self.ch() == '*' and self.ch(1) == '/':
+        self.adv()
+        self.adv()
+        return
+      self.adv()
+    self.err('Unterminated block comment')
+
+  fn tokenize():
+    var bol = true
+    while self.pos < self.src.length:
+      if bol:
+        bol = false
+        var indent = 0
+        while self.ch() == ' ' or self.ch() == '\t':
+          if self.ch() == '\t':
+            indent = indent + 4
+          else:
+            indent = indent + 1
+          self.adv()
+        if self.ch() == '\n' or not self.ch():
+          if self.ch() == '\n':
+            self.adv()
+            bol = true
+          continue
+        if self.ch() == '/' and self.ch(1) == '/':
+          while self.pos < self.src.length and self.ch() != '\n': self.adv()
+          if self.ch() == '\n':
+            self.adv()
+            bol = true
+          continue
+        if self.ch() == '/' and self.ch(1) == '*':
+          self.scanBlockComment()
+          continue
+        val isContinuation = (self.ch() == '|' and self.ch(1) == '>') or self.ch() == '.' or (self.ch() == '?' and self.ch(1) == '.')
+        if isContinuation:
+          val lastTok = self.tokens[self.tokens.length - 1]
+          if lastTok and lastTok.type == T.NEWLINE: self.tokens.pop()
+        else:
+          self.applyIndent(indent)
+
+      val l = self.line
+      val c = self.col
+      val cur = self.ch()
+
+      if cur == '\n':
+        self.adv()
+        bol = true
+        if self.nestDepth == 0:
+          val lastTok = self.tokens[self.tokens.length - 1]
+          if lastTok and lastTok.type != T.NEWLINE and lastTok.type != T.INDENT and lastTok.type != T.DEDENT:
+            self.tok(T.NEWLINE, null, l, c)
+        continue
+
+      if cur == ' ' or cur == '\t':
+        self.adv()
+        continue
+
+      if cur == '/' and self.ch(1) == '/':
+        while self.pos < self.src.length and self.ch() != '\n': self.adv()
+        continue
+
+      if cur == '/' and self.ch(1) == '*':
+        self.scanBlockComment()
+        continue
+
+      if cur >= '0' and cur <= '9':
+        if cur == '0' and (self.ch(1) == 'x' or self.ch(1) == 'X'):
+          self.adv()
+          self.adv()
+          var h = ''
+          while /[0-9a-fA-F_]/.test(self.ch()):
+            val hc = self.adv()
+            if hc != '_': h += hc
+          self.tok(T.NUMBER, parseInt(h.length > 0 ? h : '0', 16), l, c)
+          continue
+        if cur == '0' and (self.ch(1) == 'b' or self.ch(1) == 'B'):
+          self.adv()
+          self.adv()
+          var b = ''
+          while /[01_]/.test(self.ch()):
+            val bc = self.adv()
+            if bc != '_': b += bc
+          self.tok(T.NUMBER, parseInt(b.length > 0 ? b : '0', 2), l, c)
+          continue
+        var numStr = ''
+        while self.pos < self.src.length:
+          if self.ch() == '.' and self.ch(1) == '.': break
+          if (self.ch() >= '0' and self.ch() <= '9') or self.ch() == '.':
+            numStr += self.adv()
+          else if self.ch() == '_':
+            self.adv()
+          else if (self.ch() == 'e' or self.ch() == 'E') and numStr.length > 0:
+            numStr += self.adv()
+            if self.ch() == '+' or self.ch() == '-': numStr += self.adv()
+            while self.ch() >= '0' and self.ch() <= '9': numStr += self.adv()
+            break
+          else:
+            break
+        self.tok(T.NUMBER, parseFloat(numStr), l, c)
+        continue
+
+      if cur == '"':
+        self.scanStr(l, c)
+        continue
+
+      if cur == '`':
+        self.scanBacktick(l, c)
+        continue
+
+      if cur == "'":
+        self.adv()
+        var sq = ''
+        while self.pos < self.src.length and self.ch() != "'":
+          if self.ch() == '\\':
+            self.adv()
+            val e = self.adv()
+            if e == 'n': sq += '\n'
+            else if e == 't': sq += '\t'
+            else if e == 'r': sq += '\r'
+            else if e == "'": sq += "'"
+            else if e == '\\': sq += '\\'
+            else: sq += '\\' + e
+          else:
+            sq += self.adv()
+        if self.ch() != "'": self.err('Unterminated string')
+        self.adv()
+        self.tok(T.STRING, sq, l, c)
+        continue
+
+      if (cur >= 'a' and cur <= 'z') or (cur >= 'A' and cur <= 'Z') or cur == '_':
+        var word = ''
+        while /[a-zA-Z0-9_]/.test(self.ch()): word += self.adv()
+        if word == '_' and not /[a-zA-Z0-9_]/.test(self.ch()):
+          self.tok(T.WILDCARD, '_', l, c)
+          continue
+        val kw = KEYWORDS[word]
+        if kw == '__TRUE__': self.tok(T.BOOL, true, l, c)
+        else if kw == '__FALSE__': self.tok(T.BOOL, false, l, c)
+        else if kw == '__NULL__': self.tok(T.NULL, null, l, c)
+        else if kw != undefined: self.tok(kw, word, l, c)
+        else: self.tok(T.IDENT, word, l, c)
+        continue
+
+      self.adv()
+
+      if cur == '+':
+        if self.ch() == '+':
+          self.adv()
+          self.tok(T.PLUSPLUS, '++', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.PLUSEQ, '+=', l, c)
+        else:
+          self.tok(T.PLUS, '+', l, c)
+      else if cur == '-':
+        if self.ch() == '-':
+          self.adv()
+          self.tok(T.MINUSMINUS, '--', l, c)
+        else if self.ch() == '>':
+          self.adv()
+          self.tok(T.ARROW, '->', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.MINUSEQ, '-=', l, c)
+        else:
+          self.tok(T.MINUS, '-', l, c)
+      else if cur == '*':
+        if self.ch() == '*':
+          self.adv()
+          self.tok(T.STARSTAR, '**', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.STAREQ, '*=', l, c)
+        else:
+          self.tok(T.STAR, '*', l, c)
+      else if cur == '/':
+        if self.ch() == '=':
+          self.adv()
+          self.tok(T.SLASHEQ, '/=', l, c)
+        else:
+          val lastTok = self.tokens[self.tokens.length - 1]
+          val afterVal = lastTok != null and (
+            lastTok.type == T.IDENT or lastTok.type == T.NUMBER or
+            lastTok.type == T.STRING or lastTok.type == T.BOOL or
+            lastTok.type == T.NULL or lastTok.type == T.REGEX or
+            lastTok.type == T.RPAREN or lastTok.type == T.RBRACKET or
+            lastTok.type == T.PLUSPLUS or lastTok.type == T.MINUSMINUS or
+            lastTok.type == T.BANG
+          )
+          if afterVal:
+            self.tok(T.SLASH, '/', l, c)
+          else:
+            self.scanRegexBody(l, c)
+      else if cur == '%':
+        if self.ch() == '=':
+          self.adv()
+          self.tok(T.PERCENTEQ, '%=', l, c)
+        else:
+          self.tok(T.PERCENT, '%', l, c)
+      else if cur == '=':
+        if self.ch() == '=' and self.src[self.pos + 1] == '=':
+          self.adv()
+          self.adv()
+          self.tok(T.EQEQEQ, '===', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.EQEQ, '==', l, c)
+        else if self.ch() == '>':
+          self.adv()
+          self.tok(T.FATARROW, '=>', l, c)
+        else:
+          self.tok(T.EQ, '=', l, c)
+      else if cur == '!':
+        if self.ch() == '=' and self.src[self.pos + 1] == '=':
+          self.adv()
+          self.adv()
+          self.tok(T.NEQEQ, '!==', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.NEQ, '!=', l, c)
+        else:
+          self.tok(T.BANG, '!', l, c)
+      else if cur == '<':
+        if self.ch() == '<':
+          self.adv()
+          self.tok(T.LSHIFT, '<<', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.LTE, '<=', l, c)
+        else:
+          self.tok(T.LT, '<', l, c)
+      else if cur == '>':
+        if self.ch() == '>':
+          self.adv()
+          self.tok(T.RSHIFT, '>>', l, c)
+        else if self.ch() == '=':
+          self.adv()
+          self.tok(T.GTE, '>=', l, c)
+        else:
+          self.tok(T.GT, '>', l, c)
+      else if cur == '.':
+        if self.ch() == '.' and self.src[self.pos + 1] == '.':
+          self.adv()
+          self.adv()
+          self.tok(T.DOTDOTDOT, '...', l, c)
+        else if self.ch() == '.':
+          self.adv()
+          self.tok(T.DOTDOT, '..', l, c)
+        else:
+          self.tok(T.DOT, '.', l, c)
+      else if cur == '?':
+        if self.ch() == '.':
+          self.adv()
+          self.tok(T.QUESTIONDOT, '?.', l, c)
+        else if self.ch() == '?':
+          self.adv()
+          self.tok(T.NULLISH, '??', l, c)
+        else:
+          self.tok(T.QUESTION, '?', l, c)
+      else if cur == '|':
+        if self.ch() == '>':
+          self.adv()
+          self.tok(T.PIPE, '|>', l, c)
+        else if self.ch() == '|':
+          self.adv()
+          self.tok(T.OROR, '||', l, c)
+        else:
+          self.tok(T.PIPEB, '|', l, c)
+      else if cur == '&':
+        if self.ch() == '&':
+          self.adv()
+          self.tok(T.ANDAND, '&&', l, c)
+        else:
+          self.tok(T.AMPERSAND, '&', l, c)
+      else if cur == '^':
+        self.tok(T.CARET, '^', l, c)
+      else if cur == '~':
+        self.tok(T.TILDE, '~', l, c)
+      else if cur == '@':
+        self.tok(T.AT, '@', l, c)
+      else if cur == '(':
+        self.nestDepth = self.nestDepth + 1
+        self.tok(T.LPAREN, '(', l, c)
+      else if cur == ')':
+        self.nestDepth = self.nestDepth - 1
+        self.tok(T.RPAREN, ')', l, c)
+      else if cur == '[':
+        self.nestDepth = self.nestDepth + 1
+        self.tok(T.LBRACKET, '[', l, c)
+      else if cur == ']':
+        self.nestDepth = self.nestDepth - 1
+        self.tok(T.RBRACKET, ']', l, c)
+      else if cur == '{':
+        self.nestDepth = self.nestDepth + 1
+        self.tok(T.LBRACE, '{', l, c)
+      else if cur == '}':
+        self.nestDepth = self.nestDepth - 1
+        self.tok(T.RBRACE, '}', l, c)
+      else if cur == ',':
+        self.tok(T.COMMA, ',', l, c)
+      else if cur == ':':
+        self.tok(T.COLON, ':', l, c)
+      else if cur != ';':
+        self.err("Unknown character: '{cur}'")
+
+    while self.indentStack.length > 1:
+      self.indentStack.pop()
+      self.tok(T.DEDENT, null, self.line, 1)
+    val lastTok = self.tokens[self.tokens.length - 1]
+    if lastTok and lastTok.type != T.NEWLINE and lastTok.type != T.DEDENT:
+      self.tok(T.NEWLINE, null, self.line, self.col)
+    self.tok(T.EOF, null, self.line, self.col)
+    return self.tokens
+
+export fn lexerize(source):
+  val processed = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n')
+  return new Lexer(processed, 0, 1, 1, [], [0], 0)