@xnoxs/flux-lang 3.1.1

package/src/self/lexer.js

@@ -0,0 +1,709 @@
+// ── Flux stdlib ──
+
+function map(arr, fn) { return arr.map(fn); }
+
+function some(arr, fn) { return arr.some(fn); }
+
+function join(arr, sep) { return arr.join(sep != null ? sep : ','); }
+// ── end stdlib ──
+
+// Generated by Flux Transpiler v3.1.0
+"use strict";
+
+const T = { NUMBER: "NUMBER", STRING: "STRING", BOOL: "BOOL", NULL: "NULL", IDENT: "IDENT", VAR: "VAR", VAL: "VAL", FN: "FN", RETURN: "RETURN", IF: "IF", ELSE: "ELSE", FOR: "FOR", IN: "IN", WHILE: "WHILE", BREAK: "BREAK", CONTINUE: "CONTINUE", DO: "DO", CLASS: "CLASS", EXTENDS: "EXTENDS", SELF: "SELF", NEW: "NEW", INTERFACE: "INTERFACE", IMPLEMENTS: "IMPLEMENTS", PRIVATE: "PRIVATE", PUBLIC: "PUBLIC", PROTECTED: "PROTECTED", READONLY: "READONLY", STATIC: "STATIC", ABSTRACT: "ABSTRACT", OVERRIDE: "OVERRIDE", MATCH: "MATCH", WHEN: "WHEN", IMPORT: "IMPORT", EXPORT: "EXPORT", FROM: "FROM", AS: "AS", DEFAULT: "DEFAULT", AND: "AND", OR: "OR", NOT: "NOT", ASYNC: "ASYNC", AWAIT: "AWAIT", TRY: "TRY", CATCH: "CATCH", FINALLY: "FINALLY", THROW: "THROW", TYPEOF: "TYPEOF", INSTANCEOF: "INSTANCEOF", TYPE: "TYPE", ENUM: "ENUM", SATISFIES: "SATISFIES", IS: "IS", CONST: "CONST", PLUS: "PLUS", MINUS: "MINUS", STAR: "STAR", SLASH: "SLASH", PERCENT: "PERCENT", REGEX: "REGEX", STARSTAR: "STARSTAR", EQ: "EQ", EQEQ: "EQEQ", NEQ: "NEQ", EQEQEQ: "EQEQEQ", NEQEQ: "NEQEQ", LT: "LT", LTE: "LTE", GT: "GT", GTE: "GTE", PLUSEQ: "PLUSEQ", MINUSEQ: "MINUSEQ", STAREQ: "STAREQ", SLASHEQ: "SLASHEQ", PERCENTEQ: "PERCENTEQ", PLUSPLUS: "PLUSPLUS", MINUSMINUS: "MINUSMINUS", AMPERSAND: "AMPERSAND", ANDAND: "ANDAND", PIPEB: "PIPEB", OROR: "OROR", CARET: "CARET", TILDE: "TILDE", LSHIFT: "LSHIFT", RSHIFT: "RSHIFT", ARROW: "ARROW", FATARROW: "FATARROW", PIPE: "PIPE", DOTDOT: "DOTDOT", DOTDOTDOT: "DOTDOTDOT", WILDCARD: "WILDCARD", NULLISH: "NULLISH", QUESTIONDOT: "QUESTIONDOT", BANG: "BANG", AT: "AT", LPAREN: "LPAREN", RPAREN: "RPAREN", LBRACKET: "LBRACKET", RBRACKET: "RBRACKET", LBRACE: "LBRACE", RBRACE: "RBRACE", COMMA: "COMMA", DOT: "DOT", COLON: "COLON", QUESTION: "QUESTION", NEWLINE: "NEWLINE", INDENT: "INDENT", DEDENT: "DEDENT", EOF: "EOF" };
+module.exports.T = T;
+const TokenType = T;
+module.exports.TokenType = TokenType;
+const KEYWORDS = { var: "VAR", val: "VAL", fn: "FN", return: "RETURN", if: "IF", else: "ELSE", for: "FOR", in: "IN", while: "WHILE", break: "BREAK", continue: "CONTINUE", do: "DO", class: "CLASS", extends: "EXTENDS", self: "SELF", new: "NEW", interface: "INTERFACE", implements: "IMPLEMENTS", private: "PRIVATE", public: "PUBLIC", protected: "PROTECTED", readonly: "READONLY", static: "STATIC", abstract: "ABSTRACT", override: "OVERRIDE", match: "MATCH", when: "WHEN", import: "IMPORT", export: "EXPORT", from: "FROM", as: "AS", default: "DEFAULT", and: "AND", or: "OR", not: "NOT", async: "ASYNC", await: "AWAIT", try: "TRY", catch: "CATCH", finally: "FINALLY", throw: "THROW", typeof: "TYPEOF", instanceof: "INSTANCEOF", type: "TYPE", enum: "ENUM", satisfies: "SATISFIES", is: "IS", const: "CONST", true: "__TRUE__", false: "__FALSE__", null: "__NULL__" };
+class Lexer {
+  constructor(src, pos, line, col, tokens, indentStack, nestDepth) {
+    this.src = src;
+    this.pos = pos;
+    this.line = line;
+    this.col = col;
+    this.tokens = tokens;
+    this.indentStack = indentStack;
+    this.nestDepth = nestDepth;
+  }
+
+  err(msg) {
+    throw new Error(`[Lexer ${this.line}:${this.col}] ${msg}`);
+  }
+
+  ch(n = 0) {
+    return (this.src[(this.pos + n)] ?? "");
+  }
+
+  adv() {
+    const c = this.src[this.pos];
+    this.pos = (this.pos + 1);
+    if ((c == "\n")) {
+      this.line = (this.line + 1);
+      this.col = 1;
+    }
+    else {
+      this.col = (this.col + 1);
+    }
+    return c;
+  }
+
+  tok(typ, value, l, c) {
+    const v = ((value != undefined) ? value : typ);
+    const ln = ((l != undefined) ? l : this.line);
+    const co = ((c != undefined) ? c : this.col);
+    this.tokens.push({ type: typ, value: v, line: ln, col: co });
+  }
+
+  applyIndent(indent) {
+    if ((this.nestDepth > 0)) {
+      return;
+    }
+    const top = this.indentStack[(this.indentStack.length - 1)];
+    if ((indent > top)) {
+      this.indentStack.push(indent);
+      this.tok(T.INDENT, indent, this.line, 1);
+    }
+    else if ((indent < top)) {
+      while (((this.indentStack.length > 1) && (this.indentStack[(this.indentStack.length - 1)] > indent))) {
+        this.indentStack.pop();
+        this.tok(T.DEDENT, null, this.line, 1);
+      }
+      if ((this.indentStack[(this.indentStack.length - 1)] != indent)) {
+        this.err(`Inconsistent indentation (${indent} spaces)`);
+      }
+    }
+  }
+
+  scanBacktick(l, c) {
+    this.adv();
+    let s = "";
+    while (((this.pos < this.src.length) && (this.ch() != "`"))) {
+      if ((this.ch() == "\\")) {
+        this.adv();
+        const e = this.adv();
+        if ((e == "n")) {
+          s += "\n";
+        }
+        else if ((e == "t")) {
+          s += "\t";
+        }
+        else if ((e == "`")) {
+          s += "`";
+        }
+        else if ((e == "\\")) {
+          s += "\\";
+        }
+        else {
+          s += ("\\" + e);
+        }
+      }
+      else {
+        s += this.adv();
+      }
+    }
+    if ((this.ch() != "`")) {
+      this.err("Unterminated backtick string");
+    }
+    this.adv();
+    const rawLines = s.split("\n");
+    const trimmed = rawLines.map((ln) => ln.trimEnd());
+    while ((trimmed.length && !trimmed[0].trim())) {
+      trimmed.shift();
+    }
+    while ((trimmed.length && !trimmed[(trimmed.length - 1)].trim())) {
+      trimmed.pop();
+    }
+    let minIndent = 999999;
+    for (const ln of trimmed) {
+      if (ln.trim()) {
+        const m = ln.match(/^(\s*)/);
+        const ind = m[1].length;
+        if ((ind < minIndent)) {
+          minIndent = ind;
+        }
+      }
+    }
+    if ((minIndent == 999999)) {
+      minIndent = 0;
+    }
+    const dedented = trimmed.map((ln) => ln.slice(minIndent));
+    this.tok(T.STRING, dedented.join("\n"), l, c);
+  }
+
+  scanStr(l, c) {
+    this.adv();
+    const parts = [];
+    let text = "";
+    while (((this.pos < this.src.length) && (this.ch() != "\""))) {
+      if ((this.ch() == "\\")) {
+        this.adv();
+        const e = this.adv();
+        if ((e == "n")) {
+          text += "\n";
+        }
+        else if ((e == "t")) {
+          text += "\t";
+        }
+        else if ((e == "\"")) {
+          text += "\"";
+        }
+        else if ((e == "'")) {
+          text += "'";
+        }
+        else if ((e == "\\")) {
+          text += "\\";
+        }
+        else if ((e == "{")) {
+          text += "{";
+        }
+        else if ((e == "}")) {
+          text += "}";
+        }
+        else {
+          text += ("\\" + e);
+        }
+      }
+      else if ((this.ch() == "{")) {
+        parts.push({ type: "text", value: text });
+        text = "";
+        this.adv();
+        let depth = 1;
+        let expr = "";
+        while ((this.pos < this.src.length)) {
+          const cc = this.ch();
+          if ((cc == "{")) {
+            depth = (depth + 1);
+          }
+          if ((cc == "}")) {
+            depth = (depth - 1);
+            if ((depth == 0)) {
+              break;
+            }
+          }
+          if (((cc == "\\") && ((this.src[(this.pos + 1)] == "\"") || (this.src[(this.pos + 1)] == "'")))) {
+            this.adv();
+            expr += this.adv();
+          }
+          else {
+            expr += this.adv();
+          }
+        }
+        if ((this.ch() != "}")) {
+          this.err("Unclosed string interpolation");
+        }
+        this.adv();
+        parts.push({ type: "expr", value: expr.trim() });
+      }
+      else {
+        text += this.adv();
+      }
+    }
+    if ((this.ch() != "\"")) {
+      this.err("Unterminated string");
+    }
+    this.adv();
+    parts.push({ type: "text", value: text });
+    if (parts.some((p) => (p.type == "expr"))) {
+      this.tok(T.STRING, { template: true, parts }, l, c);
+    }
+    else {
+      this.tok(T.STRING, parts.map((p) => p.value).join(""), l, c);
+    }
+  }
+
+  scanRegexBody(l, c) {
+    let pattern = "";
+    let inClass = false;
+    while ((this.pos < this.src.length)) {
+      const ch = this.ch();
+      if ((ch == "\n")) {
+        this.err("Unterminated regex literal");
+      }
+      if ((ch == "\\")) {
+        pattern += this.adv();
+        if ((this.pos < this.src.length)) {
+          pattern += this.adv();
+        }
+        continue;
+      }
+      if ((ch == "[")) {
+        inClass = true;
+        pattern += this.adv();
+        continue;
+      }
+      if ((ch == "]")) {
+        inClass = false;
+        pattern += this.adv();
+        continue;
+      }
+      if (((ch == "/") && !inClass)) {
+        break;
+      }
+      pattern += this.adv();
+    }
+    if ((this.ch() != "/")) {
+      this.err("Unterminated regex literal");
+    }
+    this.adv();
+    let flags = "";
+    while (/[gimsuy]/.test(this.ch())) {
+      flags += this.adv();
+    }
+    this.tok(T.REGEX, { pattern, flags }, l, c);
+  }
+
+  scanBlockComment() {
+    this.adv();
+    this.adv();
+    while ((this.pos < this.src.length)) {
+      if (((this.ch() == "*") && (this.ch(1) == "/"))) {
+        this.adv();
+        this.adv();
+        return;
+      }
+      this.adv();
+    }
+    this.err("Unterminated block comment");
+  }
+
+  tokenize() {
+    let bol = true;
+    while ((this.pos < this.src.length)) {
+      if (bol) {
+        bol = false;
+        let indent = 0;
+        while (((this.ch() == " ") || (this.ch() == "\t"))) {
+          if ((this.ch() == "\t")) {
+            indent = (indent + 4);
+          }
+          else {
+            indent = (indent + 1);
+          }
+          this.adv();
+        }
+        if (((this.ch() == "\n") || !this.ch())) {
+          if ((this.ch() == "\n")) {
+            this.adv();
+            bol = true;
+          }
+          continue;
+        }
+        if (((this.ch() == "/") && (this.ch(1) == "/"))) {
+          while (((this.pos < this.src.length) && (this.ch() != "\n"))) {
+            this.adv();
+          }
+          if ((this.ch() == "\n")) {
+            this.adv();
+            bol = true;
+          }
+          continue;
+        }
+        if (((this.ch() == "/") && (this.ch(1) == "*"))) {
+          this.scanBlockComment();
+          continue;
+        }
+        const isContinuation = ((((this.ch() == "|") && (this.ch(1) == ">")) || (this.ch() == ".")) || ((this.ch() == "?") && (this.ch(1) == ".")));
+        if (isContinuation) {
+          const lastTok = this.tokens[(this.tokens.length - 1)];
+          if ((lastTok && (lastTok.type == T.NEWLINE))) {
+            this.tokens.pop();
+          }
+        }
+        else {
+          this.applyIndent(indent);
+        }
+      }
+      const l = this.line;
+      const c = this.col;
+      const cur = this.ch();
+      if ((cur == "\n")) {
+        this.adv();
+        bol = true;
+        if ((this.nestDepth == 0)) {
+          const lastTok = this.tokens[(this.tokens.length - 1)];
+          if ((((lastTok && (lastTok.type != T.NEWLINE)) && (lastTok.type != T.INDENT)) && (lastTok.type != T.DEDENT))) {
+            this.tok(T.NEWLINE, null, l, c);
+          }
+        }
+        continue;
+      }
+      if (((cur == " ") || (cur == "\t"))) {
+        this.adv();
+        continue;
+      }
+      if (((cur == "/") && (this.ch(1) == "/"))) {
+        while (((this.pos < this.src.length) && (this.ch() != "\n"))) {
+          this.adv();
+        }
+        continue;
+      }
+      if (((cur == "/") && (this.ch(1) == "*"))) {
+        this.scanBlockComment();
+        continue;
+      }
+      if (((cur >= "0") && (cur <= "9"))) {
+        if (((cur == "0") && ((this.ch(1) == "x") || (this.ch(1) == "X")))) {
+          this.adv();
+          this.adv();
+          let h = "";
+          while (/[0-9a-fA-F_]/.test(this.ch())) {
+            const hc = this.adv();
+            if ((hc != "_")) {
+              h += hc;
+            }
+          }
+          this.tok(T.NUMBER, parseInt(((h.length > 0) ? h : "0"), 16), l, c);
+          continue;
+        }
+        if (((cur == "0") && ((this.ch(1) == "b") || (this.ch(1) == "B")))) {
+          this.adv();
+          this.adv();
+          let b = "";
+          while (/[01_]/.test(this.ch())) {
+            const bc = this.adv();
+            if ((bc != "_")) {
+              b += bc;
+            }
+          }
+          this.tok(T.NUMBER, parseInt(((b.length > 0) ? b : "0"), 2), l, c);
+          continue;
+        }
+        let numStr = "";
+        while ((this.pos < this.src.length)) {
+          if (((this.ch() == ".") && (this.ch(1) == "."))) {
+            break;
+          }
+          if ((((this.ch() >= "0") && (this.ch() <= "9")) || (this.ch() == "."))) {
+            numStr += this.adv();
+          }
+          else if ((this.ch() == "_")) {
+            this.adv();
+          }
+          else if ((((this.ch() == "e") || (this.ch() == "E")) && (numStr.length > 0))) {
+            numStr += this.adv();
+            if (((this.ch() == "+") || (this.ch() == "-"))) {
+              numStr += this.adv();
+            }
+            while (((this.ch() >= "0") && (this.ch() <= "9"))) {
+              numStr += this.adv();
+            }
+            break;
+          }
+          else {
+            break;
+          }
+        }
+        this.tok(T.NUMBER, parseFloat(numStr), l, c);
+        continue;
+      }
+      if ((cur == "\"")) {
+        this.scanStr(l, c);
+        continue;
+      }
+      if ((cur == "`")) {
+        this.scanBacktick(l, c);
+        continue;
+      }
+      if ((cur == "'")) {
+        this.adv();
+        let sq = "";
+        while (((this.pos < this.src.length) && (this.ch() != "'"))) {
+          if ((this.ch() == "\\")) {
+            this.adv();
+            const e = this.adv();
+            if ((e == "n")) {
+              sq += "\n";
+            }
+            else if ((e == "t")) {
+              sq += "\t";
+            }
+            else if ((e == "r")) {
+              sq += "\r";
+            }
+            else if ((e == "'")) {
+              sq += "'";
+            }
+            else if ((e == "\\")) {
+              sq += "\\";
+            }
+            else {
+              sq += ("\\" + e);
+            }
+          }
+          else {
+            sq += this.adv();
+          }
+        }
+        if ((this.ch() != "'")) {
+          this.err("Unterminated string");
+        }
+        this.adv();
+        this.tok(T.STRING, sq, l, c);
+        continue;
+      }
+      if (((((cur >= "a") && (cur <= "z")) || ((cur >= "A") && (cur <= "Z"))) || (cur == "_"))) {
+        let word = "";
+        while (/[a-zA-Z0-9_]/.test(this.ch())) {
+          word += this.adv();
+        }
+        if (((word == "_") && !/[a-zA-Z0-9_]/.test(this.ch()))) {
+          this.tok(T.WILDCARD, "_", l, c);
+          continue;
+        }
+        const kw = KEYWORDS[word];
+        if ((kw == "__TRUE__")) {
+          this.tok(T.BOOL, true, l, c);
+        }
+        else if ((kw == "__FALSE__")) {
+          this.tok(T.BOOL, false, l, c);
+        }
+        else if ((kw == "__NULL__")) {
+          this.tok(T.NULL, null, l, c);
+        }
+        else if ((kw != undefined)) {
+          this.tok(kw, word, l, c);
+        }
+        else {
+          this.tok(T.IDENT, word, l, c);
+        }
+        continue;
+      }
+      this.adv();
+      if ((cur == "+")) {
+        if ((this.ch() == "+")) {
+          this.adv();
+          this.tok(T.PLUSPLUS, "++", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.PLUSEQ, "+=", l, c);
+        }
+        else {
+          this.tok(T.PLUS, "+", l, c);
+        }
+      }
+      else if ((cur == "-")) {
+        if ((this.ch() == "-")) {
+          this.adv();
+          this.tok(T.MINUSMINUS, "--", l, c);
+        }
+        else if ((this.ch() == ">")) {
+          this.adv();
+          this.tok(T.ARROW, "->", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.MINUSEQ, "-=", l, c);
+        }
+        else {
+          this.tok(T.MINUS, "-", l, c);
+        }
+      }
+      else if ((cur == "*")) {
+        if ((this.ch() == "*")) {
+          this.adv();
+          this.tok(T.STARSTAR, "**", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.STAREQ, "*=", l, c);
+        }
+        else {
+          this.tok(T.STAR, "*", l, c);
+        }
+      }
+      else if ((cur == "/")) {
+        if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.SLASHEQ, "/=", l, c);
+        }
+        else {
+          const lastTok = this.tokens[(this.tokens.length - 1)];
+          const afterVal = ((lastTok != null) && (((((((((((lastTok.type == T.IDENT) || (lastTok.type == T.NUMBER)) || (lastTok.type == T.STRING)) || (lastTok.type == T.BOOL)) || (lastTok.type == T.NULL)) || (lastTok.type == T.REGEX)) || (lastTok.type == T.RPAREN)) || (lastTok.type == T.RBRACKET)) || (lastTok.type == T.PLUSPLUS)) || (lastTok.type == T.MINUSMINUS)) || (lastTok.type == T.BANG)));
+          if (afterVal) {
+            this.tok(T.SLASH, "/", l, c);
+          }
+          else {
+            this.scanRegexBody(l, c);
+          }
+        }
+      }
+      else if ((cur == "%")) {
+        if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.PERCENTEQ, "%=", l, c);
+        }
+        else {
+          this.tok(T.PERCENT, "%", l, c);
+        }
+      }
+      else if ((cur == "=")) {
+        if (((this.ch() == "=") && (this.src[(this.pos + 1)] == "="))) {
+          this.adv();
+          this.adv();
+          this.tok(T.EQEQEQ, "===", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.EQEQ, "==", l, c);
+        }
+        else if ((this.ch() == ">")) {
+          this.adv();
+          this.tok(T.FATARROW, "=>", l, c);
+        }
+        else {
+          this.tok(T.EQ, "=", l, c);
+        }
+      }
+      else if ((cur == "!")) {
+        if (((this.ch() == "=") && (this.src[(this.pos + 1)] == "="))) {
+          this.adv();
+          this.adv();
+          this.tok(T.NEQEQ, "!==", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.NEQ, "!=", l, c);
+        }
+        else {
+          this.tok(T.BANG, "!", l, c);
+        }
+      }
+      else if ((cur == "<")) {
+        if ((this.ch() == "<")) {
+          this.adv();
+          this.tok(T.LSHIFT, "<<", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.LTE, "<=", l, c);
+        }
+        else {
+          this.tok(T.LT, "<", l, c);
+        }
+      }
+      else if ((cur == ">")) {
+        if ((this.ch() == ">")) {
+          this.adv();
+          this.tok(T.RSHIFT, ">>", l, c);
+        }
+        else if ((this.ch() == "=")) {
+          this.adv();
+          this.tok(T.GTE, ">=", l, c);
+        }
+        else {
+          this.tok(T.GT, ">", l, c);
+        }
+      }
+      else if ((cur == ".")) {
+        if (((this.ch() == ".") && (this.src[(this.pos + 1)] == "."))) {
+          this.adv();
+          this.adv();
+          this.tok(T.DOTDOTDOT, "...", l, c);
+        }
+        else if ((this.ch() == ".")) {
+          this.adv();
+          this.tok(T.DOTDOT, "..", l, c);
+        }
+        else {
+          this.tok(T.DOT, ".", l, c);
+        }
+      }
+      else if ((cur == "?")) {
+        if ((this.ch() == ".")) {
+          this.adv();
+          this.tok(T.QUESTIONDOT, "?.", l, c);
+        }
+        else if ((this.ch() == "?")) {
+          this.adv();
+          this.tok(T.NULLISH, "??", l, c);
+        }
+        else {
+          this.tok(T.QUESTION, "?", l, c);
+        }
+      }
+      else if ((cur == "|")) {
+        if ((this.ch() == ">")) {
+          this.adv();
+          this.tok(T.PIPE, "|>", l, c);
+        }
+        else if ((this.ch() == "|")) {
+          this.adv();
+          this.tok(T.OROR, "||", l, c);
+        }
+        else {
+          this.tok(T.PIPEB, "|", l, c);
+        }
+      }
+      else if ((cur == "&")) {
+        if ((this.ch() == "&")) {
+          this.adv();
+          this.tok(T.ANDAND, "&&", l, c);
+        }
+        else {
+          this.tok(T.AMPERSAND, "&", l, c);
+        }
+      }
+      else if ((cur == "^")) {
+        this.tok(T.CARET, "^", l, c);
+      }
+      else if ((cur == "~")) {
+        this.tok(T.TILDE, "~", l, c);
+      }
+      else if ((cur == "@")) {
+        this.tok(T.AT, "@", l, c);
+      }
+      else if ((cur == "(")) {
+        this.nestDepth = (this.nestDepth + 1);
+        this.tok(T.LPAREN, "(", l, c);
+      }
+      else if ((cur == ")")) {
+        this.nestDepth = (this.nestDepth - 1);
+        this.tok(T.RPAREN, ")", l, c);
+      }
+      else if ((cur == "[")) {
+        this.nestDepth = (this.nestDepth + 1);
+        this.tok(T.LBRACKET, "[", l, c);
+      }
+      else if ((cur == "]")) {
+        this.nestDepth = (this.nestDepth - 1);
+        this.tok(T.RBRACKET, "]", l, c);
+      }
+      else if ((cur == "{")) {
+        this.nestDepth = (this.nestDepth + 1);
+        this.tok(T.LBRACE, "{", l, c);
+      }
+      else if ((cur == "}")) {
+        this.nestDepth = (this.nestDepth - 1);
+        this.tok(T.RBRACE, "}", l, c);
+      }
+      else if ((cur == ",")) {
+        this.tok(T.COMMA, ",", l, c);
+      }
+      else if ((cur == ":")) {
+        this.tok(T.COLON, ":", l, c);
+      }
+      else if ((cur != ";")) {
+        this.err(`Unknown character: '${cur}'`);
+      }
+    }
+    while ((this.indentStack.length > 1)) {
+      this.indentStack.pop();
+      this.tok(T.DEDENT, null, this.line, 1);
+    }
+    const lastTok = this.tokens[(this.tokens.length - 1)];
+    if (((lastTok && (lastTok.type != T.NEWLINE)) && (lastTok.type != T.DEDENT))) {
+      this.tok(T.NEWLINE, null, this.line, this.col);
+    }
+    this.tok(T.EOF, null, this.line, this.col);
+    return this.tokens;
+  }
+
+}
+
+module.exports.Lexer = Lexer;
+function lexerize(source) {
+  const processed = source.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
+  return new Lexer(processed, 0, 1, 1, [], [0], 0);
+}
+module.exports.lexerize = lexerize;