@nova-lang/cli 0.1.1 → 0.2.0

package/lib/lexer.js

@@ -0,0 +1,315 @@
+const { TokenType, Token } = require("./types");
+
+const ESCAPE_MAP = {
+  n: "\n",
+  t: "\t",
+  r: "\r",
+  '"': '"',
+  "'": "'",
+  "\\": "\\",
+  "#": "#",
+  $: "$",
+};
+
+class LexerError extends Error {
+  constructor(msg, line, col) {
+    super(`SyntaxError: ${msg} at line ${line}, col ${col}`);
+    this.name = "LexerError";
+  }
+}
+
+class Lexer {
+  constructor(source) {
+    this.source = source;
+    this.pos = 0;
+    this.line = 1;
+    this.col = 1;
+    this.tokens = [];
+    this.indentStack = [0];
+    this.lineStart = true;
+  }
+
+  peek(offset = 0) {
+    const idx = this.pos + offset;
+    return idx < this.source.length ? this.source[idx] : "\0";
+  }
+
+  advance() {
+    const ch = this.source[this.pos++];
+    if (ch === "\n") {
+      this.line++;
+      this.col = 1;
+    } else {
+      this.col++;
+    }
+    return ch;
+  }
+
+  error(msg) {
+    throw new LexerError(msg, this.line, this.col);
+  }
+
+  emit(type, value = "") {
+    this.tokens.push(new Token(type, value, this.line, this.col));
+  }
+
+  tokenize() {
+    while (this.pos < this.source.length) {
+      if (this.lineStart) {
+        this.lineStart = false;
+        this.handleIndent();
+        if (this.pos >= this.source.length) break;
+      }
+
+      const ch = this.peek(0);
+
+      if (ch === "\n") {
+        this.pos++;
+        this.line++;
+        this.col = 1;
+        this.lineStart = true;
+        this.emit(TokenType.newline);
+        continue;
+      }
+
+      if (ch === "/" && (this.peek(1) === "/" || this.peek(1) === "*")) {
+        this.skipComment();
+        continue;
+      }
+
+      if (ch === " " || ch === "\t") {
+        this.pos++;
+        this.col++;
+        continue;
+      }
+
+      if (ch === "$") {
+        if (this.peek(1) === "$") {
+          this.pos += 2;
+          const start = this.pos;
+          while (this.pos < this.source.length) {
+            if (this.peek(0) === "$" && this.peek(1) === "$") {
+              this.emit(TokenType.math_display, this.source.slice(start, this.pos));
+              this.pos += 2;
+              break;
+            }
+            this.pos++;
+          }
+          if (this.pos >= this.source.length) this.error("Unterminated display math");
+        } else {
+          this.pos++;
+          const start = this.pos;
+          while (this.pos < this.source.length && this.source[this.pos] !== "$") {
+            this.pos++;
+          }
+          if (this.pos >= this.source.length) this.error("Unterminated inline math");
+          this.emit(TokenType.math_inline, this.source.slice(start, this.pos));
+          this.pos++;
+        }
+        continue;
+      }
+
+      if (ch === "@") {
+        this.pos++;
+        this.emit(TokenType.at);
+        continue;
+      }
+
+      if (ch === "#") {
+        if (this.peek(1) === "{") {
+          this.pos += 2;
+          let depth = 1;
+          const start = this.pos;
+          while (this.pos < this.source.length && depth > 0) {
+            const c = this.source[this.pos++];
+            if (c === "{") depth++;
+            else if (c === "}") depth--;
+          }
+          this.emit(TokenType.interp_start, this.source.slice(start, this.pos - 1));
+          continue;
+        }
+      }
+
+      if (ch === '"' || ch === "'") {
+        this.pos++;
+        this.readStringInterp(ch);
+        continue;
+      }
+
+      if (ch === "(") { this.pos++; this.emit(TokenType.lparen); continue; }
+      if (ch === ")") { this.pos++; this.emit(TokenType.rparen); continue; }
+      if (ch === "{") { this.pos++; this.emit(TokenType.lbrace); continue; }
+      if (ch === "}") { this.pos++; this.emit(TokenType.rbrace); continue; }
+      if (ch === "[") { this.pos++; this.emit(TokenType.lbracket); continue; }
+      if (ch === "]") { this.pos++; this.emit(TokenType.rbracket); continue; }
+      if (ch === ":") { this.pos++; this.emit(TokenType.colon, ":"); continue; }
+      if (ch === ",") { this.pos++; this.emit(TokenType.comma); continue; }
+      if (ch === "-") { this.pos++; this.emit(TokenType.dash, "-"); continue; }
+      if (ch === "*") { this.pos++; this.emit(TokenType.star, "*"); continue; }
+      if (ch === "+") { this.pos++; this.emit(TokenType.plus, "+"); continue; }
+      if (ch === "|") { this.pos++; this.emit(TokenType.pipe, "|"); continue; }
+      if (ch === "=") { this.pos++; this.emit(TokenType.equals, "="); continue; }
+      if (ch === ".") { this.pos++; this.emit(TokenType.dot, "."); continue; }
+      if (ch === "?") { this.pos++; this.emit(TokenType.question, "?"); continue; }
+      if (ch === "<") { this.pos++; this.emit(TokenType.lt, "<"); continue; }
+      if (ch === ">") { this.pos++; this.emit(TokenType.gt, ">"); continue; }
+
+      if (/\d/.test(ch) || ((ch === "+" || ch === "-") && this.peek(1) !== "\0" && /\d/.test(this.peek(1)))) {
+        this.emit(TokenType.number, this.readNumber());
+        continue;
+      }
+
+      if (/[a-zA-Z_]/.test(ch)) {
+        const ident = this.readIdent();
+        if (ident === "true" || ident === "false") {
+          this.emit(TokenType.bool, ident);
+        } else if (ident === "null") {
+          this.emit(TokenType.null, ident);
+        } else {
+          this.emit(TokenType.ident, ident);
+        }
+        continue;
+      }
+
+      const text = this.readText();
+      if (text.length > 0) {
+        this.emit(TokenType.text, text);
+        continue;
+      }
+
+      if (ch === "\r") { this.pos++; continue; }
+      this.pos++;
+    }
+
+    while (this.indentStack.length > 1) {
+      this.indentStack.pop();
+      this.emit(TokenType.dedent);
+    }
+    this.emit(TokenType.eof);
+  }
+
+  skipComment() {
+    this.pos++;
+    if (this.peek(0) === "/") {
+      this.pos++;
+      while (this.pos < this.source.length && this.source[this.pos] !== "\n") this.pos++;
+    } else if (this.peek(0) === "*") {
+      this.pos++;
+      let depth = 1;
+      while (depth > 0 && this.pos < this.source.length) {
+        if (this.peek(0) === "/" && this.peek(1) === "*") { this.pos += 2; depth++; }
+        else if (this.peek(0) === "*" && this.peek(1) === "/") { this.pos += 2; depth--; }
+        else this.pos++;
+      }
+    }
+  }
+
+  readStringInterp(quote) {
+    while (true) {
+      let buf = "";
+      while (this.pos < this.source.length) {
+        const ch = this.advance();
+        if (ch === "\\") {
+          if (this.pos < this.source.length) {
+            const next = this.advance();
+            const escaped = ESCAPE_MAP[next];
+            buf += "\\";
+            buf += escaped !== undefined ? escaped : next;
+          }
+        } else if (ch === "#" && this.peek(0) === "{") {
+          this.pos++;
+          if (buf.length > 0) this.emit(TokenType.string, buf);
+          let depth = 1;
+          const start = this.pos;
+          while (this.pos < this.source.length && depth > 0) {
+            const c = this.source[this.pos++];
+            if (c === "{") depth++;
+            else if (c === "}") depth--;
+          }
+          this.emit(TokenType.interp_start, this.source.slice(start, this.pos - 1));
+          break;
+        } else if (ch === quote) {
+          if (buf.length > 0) this.emit(TokenType.string, buf);
+          return;
+        } else {
+          buf += ch;
+        }
+      }
+      if (this.pos >= this.source.length) return;
+    }
+  }
+
+  readNumber() {
+    const start = this.pos;
+    if (this.peek(0) === "+" || this.peek(0) === "-") this.pos++;
+    if (this.peek(0) === "0" && /[xX]/.test(this.peek(1))) {
+      this.pos += 2;
+      while (this.pos < this.source.length && /[0-9a-fA-F]/.test(this.source[this.pos])) this.pos++;
+      return this.source.slice(start, this.pos);
+    }
+    while (this.pos < this.source.length && /[0-9.]/.test(this.source[this.pos])) this.pos++;
+    if (this.pos < this.source.length && /[eE]/.test(this.source[this.pos])) {
+      this.pos++;
+      if (this.pos < this.source.length && /[+-]/.test(this.source[this.pos])) this.pos++;
+      while (this.pos < this.source.length && /\d/.test(this.source[this.pos])) this.pos++;
+    }
+    return this.source.slice(start, this.pos);
+  }
+
+  readIdent() {
+    const start = this.pos;
+    while (this.pos < this.source.length && /[a-zA-Z0-9_-]/.test(this.source[this.pos])) this.pos++;
+    return this.source.slice(start, this.pos);
+  }
+
+  readText() {
+    const start = this.pos;
+    while (this.pos < this.source.length) {
+      const ch = this.source[this.pos];
+      const next = this.pos + 1 < this.source.length ? this.source[this.pos + 1] : "\0";
+      if ("@{}$\n\"'".indexOf(ch) !== -1) break;
+      if (ch === "{" || ch === "}") break;
+      if (ch === "/" && (next === "/" || next === "*")) break;
+      if (ch === "#" && next === "{") break;
+      this.pos++;
+    }
+    return this.source.slice(start, this.pos);
+  }
+
+  handleIndent() {
+    let count = 0;
+    while (this.pos < this.source.length && this.source[this.pos] === " ") {
+      this.pos++;
+      count++;
+    }
+    while (this.pos < this.source.length && this.source[this.pos] === "\t") {
+      this.pos++;
+      count += 2;
+    }
+    if (this.pos >= this.source.length) return;
+    if (this.source[this.pos] === "\n") return;
+    if (this.source[this.pos] === "#") return;
+
+    const top = this.indentStack[this.indentStack.length - 1];
+    if (count > top) {
+      this.indentStack.push(count);
+      this.emit(TokenType.indent);
+    } else if (count < top) {
+      while (this.indentStack.length > 0 && count < this.indentStack[this.indentStack.length - 1]) {
+        this.indentStack.pop();
+        this.emit(TokenType.dedent);
+      }
+      if (this.indentStack.length > 0 && count !== this.indentStack[this.indentStack.length - 1]) {
+        this.error("Inconsistent indentation");
+      }
+    }
+  }
+}
+
+function tokenize(source) {
+  const lexer = new Lexer(source);
+  lexer.tokenize();
+  return lexer.tokens;
+}
+
+module.exports = { Lexer, tokenize };