gofront 0.0.9 → 1.0.1

package/src/lexer.js

@@ -1,99 +1,12 @@
 // Lexer: tokenizes GoFront source into a flat token stream.
 // Implements Go-style automatic semicolon insertion.
+//
+// `T` and `Token` live in ./tokens.js (a pure-data leaf module shared by
+// lexer, parser, and codegen). They are re-exported here for backward compat.
 
-export const T = {
-	// Literals
-	INT: "INT",
-	FLOAT: "FLOAT",
-	STRING: "STRING",
-	IDENT: "IDENT",
-
-	IMAG: "IMAG",
-
-	// Keywords
-	FUNC: "func",
-	VAR: "var",
-	CONST: "const",
-	TYPE: "type",
-	STRUCT: "struct",
-	INTERFACE: "interface",
-	IF: "if",
-	ELSE: "else",
-	FOR: "for",
-	RANGE: "range",
-	RETURN: "return",
-	PACKAGE: "package",
-	IMPORT: "import",
-	TRUE: "true",
-	FALSE: "false",
-	NIL: "nil",
-	BREAK: "break",
-	CONTINUE: "continue",
-	SWITCH: "switch",
-	CASE: "case",
-	DEFAULT: "default",
-	MAP: "map",
-	NEW: "new",
-	DEFER: "defer",
-	ASYNC: "async",
-	AWAIT: "await",
-	GO: "go",
-	CHAN: "chan",
-	SELECT: "select",
-
-	// Operators
-	PLUS: "+",
-	MINUS: "-",
-	STAR: "*",
-	SLASH: "/",
-	PERCENT: "%",
-	EQ: "==",
-	NEQ: "!=",
-	LT: "<",
-	GT: ">",
-	LTE: "<=",
-	GTE: ">=",
-	AND: "&&",
-	OR: "||",
-	NOT: "!",
-	ASSIGN: "=",
-	DEFINE: ":=",
-	PLUS_ASSIGN: "+=",
-	MINUS_ASSIGN: "-=",
-	STAR_ASSIGN: "*=",
-	SLASH_ASSIGN: "/=",
-	PERCENT_ASSIGN: "%=",
-	AMP_ASSIGN: "&=",
-	PIPE_ASSIGN: "|=",
-	CARET_ASSIGN: "^=",
-	LSHIFT_ASSIGN: "<<=",
-	RSHIFT_ASSIGN: ">>=",
-	INC: "++",
-	DEC: "--",
-	AMP: "&",
-	PIPE: "|",
-	CARET: "^",
-	AND_NOT: "&^",
-	TILDE: "~",
-	LSHIFT: "<<",
-	RSHIFT: ">>",
-	ELLIPSIS: "...",
-	FALLTHROUGH: "fallthrough",
-
-	// Delimiters
-	LPAREN: "(",
-	RPAREN: ")",
-	LBRACE: "{",
-	RBRACE: "}",
-	LBRACKET: "[",
-	RBRACKET: "]",
-	COMMA: ",",
-	DOT: ".",
-	COLON: ":",
-	SEMICOLON: ";",
-
-	EOF: "EOF",
-};
+import { T, Token } from "./tokens.js";
+
+export { T, Token };
 
 // Tokens where a trailing newline inserts a semicolon (Go spec §Semicolons)
 const SEMI_TRIGGERS = new Set([
@@ -149,17 +62,29 @@ const KEYWORDS = new Set([
 	"select",
 ]);
 
-export class Token {
-	constructor(type, value, line, col) {
-		this.type = type;
-		this.value = value;
-		this.line = line;
-		this.col = col;
-	}
-	toString() {
-		return `Token(${this.type}, ${JSON.stringify(this.value)}, ${this.line}:${this.col})`;
-	}
-}
+// Single-character punctuation: push a fixed token with no lookahead.
+const PUNCT_TOKEN = {
+	"~": [T.TILDE, "~"],
+	"(": [T.LPAREN, "("],
+	")": [T.RPAREN, ")"],
+	"{": [T.LBRACE, "{"],
+	"}": [T.RBRACE, "}"],
+	"[": [T.LBRACKET, "["],
+	"]": [T.RBRACKET, "]"],
+	",": [T.COMMA, ","],
+	";": [T.SEMICOLON, ";"],
+};
+
+// Operator + optional "=" compound form: [compoundType, compoundVal, baseType, baseVal]
+const ASSIGN_OP = {
+	"*": [T.STAR_ASSIGN, "*=", T.STAR, "*"],
+	"/": [T.SLASH_ASSIGN, "/=", T.SLASH, "/"],
+	"%": [T.PERCENT_ASSIGN, "%=", T.PERCENT, "%"],
+	"=": [T.EQ, "==", T.ASSIGN, "="],
+	"!": [T.NEQ, "!=", T.NOT, "!"],
+	"^": [T.CARET_ASSIGN, "^=", T.CARET, "^"],
+	":": [T.DEFINE, ":=", T.COLON, ":"],
+};
 
 export class LexError extends Error {
 	constructor(msg, line, col, filename, sourceCode) {
@@ -374,70 +299,83 @@ export class Lexer {
 	}
 
 	readNumber() {
-		let n = "";
-		let isFloat = false;
-
-		// Check for prefix: 0b, 0o, 0x
 		const first = this.advance();
-		n += first;
+		const n = first;
 		if (first === "0") {
 			const p = this.peek();
-			if (p === "b" || p === "B") {
-				n += this.advance();
-				while (this.pos < this.src.length && /[01_]/.test(this.peek())) {
-					const ch = this.advance();
-					if (ch !== "_") n += ch;
-				}
-				return { n, isFloat: false };
-			}
-			if (p === "o" || p === "O") {
-				n += this.advance();
-				while (this.pos < this.src.length && /[0-7_]/.test(this.peek())) {
-					const ch = this.advance();
-					if (ch !== "_") n += ch;
-				}
-				return { n, isFloat: false };
-			}
-			if (p === "x" || p === "X") {
-				n += this.advance();
-				while (this.pos < this.src.length && /[0-9a-fA-F_]/.test(this.peek())) {
-					const ch = this.advance();
-					if (ch !== "_") n += ch;
-				}
-				// Hex float: 0x1.Fp10 or 0xAp-2
-				if (this.peek() === "." || this.peek() === "p" || this.peek() === "P") {
-					let frac = "";
-					if (this.peek() === ".") {
-						this.advance(); // skip the dot (don't add to n)
-						while (
-							this.pos < this.src.length &&
-							/[0-9a-fA-F_]/.test(this.peek())
-						) {
-							const ch = this.advance();
-							if (ch !== "_") frac += ch;
-						}
-					}
-					if (this.peek() === "p" || this.peek() === "P") {
-						this.advance(); // skip p/P
-						let exp = "";
-						if (this.peek() === "+" || this.peek() === "-")
-							exp += this.advance();
-						while (this.pos < this.src.length && /[0-9]/.test(this.peek()))
-							exp += this.advance();
-						// Evaluate: JS can't parse hex floats, so convert manually
-						// n is "0x1" (integer hex part), frac is hex fractional digits
-						let mantissa = Number(n);
-						if (frac) {
-							mantissa += Number.parseInt(frac, 16) / 16 ** frac.length;
-						}
-						const value = mantissa * 2 ** Number(exp);
-						return { n: String(value), isFloat: true };
-					}
-				}
-				return { n, isFloat: false };
-			}
+			if (p === "b" || p === "B") return this._readBinaryNumber(n);
+			if (p === "o" || p === "O") return this._readOctalNumber(n);
+			if (p === "x" || p === "X") return this._readHexNumber(n);
+		}
+		return this._readDecimalNumber(n);
+	}
+
+	_readBinaryNumber(n) {
+		n += this.advance(); // consume b/B
+		while (this.pos < this.src.length && /[01_]/.test(this.peek())) {
+			const ch = this.advance();
+			if (ch !== "_") n += ch;
+		}
+		return { n, isFloat: false };
+	}
+
+	_readOctalNumber(n) {
+		n += this.advance(); // consume o/O
+		while (this.pos < this.src.length && /[0-7_]/.test(this.peek())) {
+			const ch = this.advance();
+			if (ch !== "_") n += ch;
+		}
+		return { n, isFloat: false };
+	}
+
+	_readHexFrac() {
+		let frac = "";
+		this.advance(); // skip the dot
+		while (this.pos < this.src.length && /[0-9a-fA-F_]/.test(this.peek())) {
+			const ch = this.advance();
+			if (ch !== "_") frac += ch;
+		}
+		return frac;
+	}
+
+	_readHexFloatExp(n, frac) {
+		this.advance(); // skip p/P
+		let exp = "";
+		if (this.peek() === "+" || this.peek() === "-") exp += this.advance();
+		while (this.pos < this.src.length && /[0-9]/.test(this.peek()))
+			exp += this.advance();
+		let mantissa = Number(n);
+		if (frac) mantissa += Number.parseInt(frac, 16) / 16 ** frac.length;
+		return { n: String(mantissa * 2 ** Number(exp)), isFloat: true };
+	}
+
+	_readHexNumber(n) {
+		n += this.advance(); // consume x/X
+		while (this.pos < this.src.length && /[0-9a-fA-F_]/.test(this.peek())) {
+			const ch = this.advance();
+			if (ch !== "_") n += ch;
 		}
+		if (this.peek() === "." || this.peek() === "p" || this.peek() === "P") {
+			let frac = "";
+			if (this.peek() === ".") frac = this._readHexFrac();
+			if (this.peek() === "p" || this.peek() === "P")
+				return this._readHexFloatExp(n, frac);
+		}
+		return { n, isFloat: false };
+	}
+
+	_readDecimalExponent(n) {
+		n += this.advance(); // consume e/E
+		if (this.peek() === "+" || this.peek() === "-") n += this.advance();
+		while (this.pos < this.src.length && /[0-9_]/.test(this.peek())) {
+			const ch = this.advance();
+			if (ch !== "_") n += ch;
+		}
+		return n;
+	}
 
+	_readDecimalNumber(n) {
+		let isFloat = false;
 		while (this.pos < this.src.length && /[0-9_]/.test(this.peek())) {
 			const ch = this.advance();
 			if (ch !== "_") n += ch;
@@ -452,12 +390,7 @@ export class Lexer {
 		}
 		if (this.peek() === "e" || this.peek() === "E") {
 			isFloat = true;
-			n += this.advance();
-			if (this.peek() === "+" || this.peek() === "-") n += this.advance();
-			while (this.pos < this.src.length && /[0-9_]/.test(this.peek())) {
-				const ch = this.advance();
-				if (ch !== "_") n += ch;
-			}
+			n = this._readDecimalExponent(n);
 		}
 		return { n, isFloat };
 	}
@@ -534,113 +467,94 @@ export class Lexer {
 
 			// Operators & punctuation
 			this.advance();
-			switch (ch) {
-				case "+":
-					if (this.match("+")) this.push(T.INC, "++", l, c);
-					else if (this.match("=")) this.push(T.PLUS_ASSIGN, "+=", l, c);
-					else this.push(T.PLUS, "+", l, c);
-					break;
-				case "-":
-					if (this.match("-")) this.push(T.DEC, "--", l, c);
-					else if (this.match("=")) this.push(T.MINUS_ASSIGN, "-=", l, c);
-					else this.push(T.MINUS, "-", l, c);
-					break;
-				case "*":
-					if (this.match("=")) this.push(T.STAR_ASSIGN, "*=", l, c);
-					else this.push(T.STAR, "*", l, c);
-					break;
-				case "/":
-					if (this.match("=")) this.push(T.SLASH_ASSIGN, "/=", l, c);
-					else this.push(T.SLASH, "/", l, c);
-					break;
-				case "%":
-					if (this.match("=")) this.push(T.PERCENT_ASSIGN, "%=", l, c);
-					else this.push(T.PERCENT, "%", l, c);
-					break;
-				case "=":
-					if (this.match("=")) this.push(T.EQ, "==", l, c);
-					else this.push(T.ASSIGN, "=", l, c);
-					break;
-				case "!":
-					if (this.match("=")) this.push(T.NEQ, "!=", l, c);
-					else this.push(T.NOT, "!", l, c);
-					break;
-				case "<":
-					if (this.match("<")) {
-						if (this.match("=")) this.push(T.LSHIFT_ASSIGN, "<<=", l, c);
-						else this.push(T.LSHIFT, "<<", l, c);
-					} else if (this.match("=")) this.push(T.LTE, "<=", l, c);
-					else this.push(T.LT, "<", l, c);
-					break;
-				case ">":
-					if (this.match(">")) {
-						if (this.match("=")) this.push(T.RSHIFT_ASSIGN, ">>=", l, c);
-						else this.push(T.RSHIFT, ">>", l, c);
-					} else if (this.match("=")) this.push(T.GTE, ">=", l, c);
-					else this.push(T.GT, ">", l, c);
-					break;
-				case "&":
-					if (this.match("&")) this.push(T.AND, "&&", l, c);
-					else if (this.match("^")) this.push(T.AND_NOT, "&^", l, c);
-					else if (this.match("=")) this.push(T.AMP_ASSIGN, "&=", l, c);
-					else this.push(T.AMP, "&", l, c);
-					break;
-				case "|":
-					if (this.match("|")) this.push(T.OR, "||", l, c);
-					else if (this.match("=")) this.push(T.PIPE_ASSIGN, "|=", l, c);
-					else this.push(T.PIPE, "|", l, c);
-					break;
-				case "^":
-					if (this.match("=")) this.push(T.CARET_ASSIGN, "^=", l, c);
-					else this.push(T.CARET, "^", l, c);
-					break;
-				case "~":
-					this.push(T.TILDE, "~", l, c);
-					break;
-				case ":":
-					if (this.match("=")) this.push(T.DEFINE, ":=", l, c);
-					else this.push(T.COLON, ":", l, c);
-					break;
-				case ".":
-					if (this.peek() === "." && this.peek(1) === ".") {
-						this.advance();
-						this.advance();
-						this.push(T.ELLIPSIS, "...", l, c);
-					} else {
-						this.push(T.DOT, ".", l, c);
-					}
-					break;
-				case "(":
-					this.push(T.LPAREN, "(", l, c);
-					break;
-				case ")":
-					this.push(T.RPAREN, ")", l, c);
-					break;
-				case "{":
-					this.push(T.LBRACE, "{", l, c);
-					break;
-				case "}":
-					this.push(T.RBRACE, "}", l, c);
-					break;
-				case "[":
-					this.push(T.LBRACKET, "[", l, c);
-					break;
-				case "]":
-					this.push(T.RBRACKET, "]", l, c);
-					break;
-				case ",":
-					this.push(T.COMMA, ",", l, c);
-					break;
-				case ";":
-					this.push(T.SEMICOLON, ";", l, c);
-					break;
-				default:
-					this.err(`Unexpected character: '${ch}'`);
-			}
+			this._readOperator(ch, l, c);
 		}
 
 		if (this.shouldSemi()) this.push(T.SEMICOLON, ";", this.line, this.col);
 		this.push(T.EOF, "", this.line, this.col);
 		return this.tokens;
 	}
+	_readPlusOp(l, c) {
+		if (this.match("+")) this.push(T.INC, "++", l, c);
+		else if (this.match("=")) this.push(T.PLUS_ASSIGN, "+=", l, c);
+		else this.push(T.PLUS, "+", l, c);
+	}
+
+	_readMinusOp(l, c) {
+		if (this.match("-")) this.push(T.DEC, "--", l, c);
+		else if (this.match("=")) this.push(T.MINUS_ASSIGN, "-=", l, c);
+		else this.push(T.MINUS, "-", l, c);
+	}
+
+	_readLessOp(l, c) {
+		if (this.match("<")) {
+			if (this.match("=")) this.push(T.LSHIFT_ASSIGN, "<<=", l, c);
+			else this.push(T.LSHIFT, "<<", l, c);
+		} else if (this.match("=")) this.push(T.LTE, "<=", l, c);
+		else this.push(T.LT, "<", l, c);
+	}
+
+	_readGreaterOp(l, c) {
+		if (this.match(">")) {
+			if (this.match("=")) this.push(T.RSHIFT_ASSIGN, ">>=", l, c);
+			else this.push(T.RSHIFT, ">>", l, c);
+		} else if (this.match("=")) this.push(T.GTE, ">=", l, c);
+		else this.push(T.GT, ">", l, c);
+	}
+
+	_readAmpOp(l, c) {
+		if (this.match("&")) this.push(T.AND, "&&", l, c);
+		else if (this.match("^")) this.push(T.AND_NOT, "&^", l, c);
+		else if (this.match("=")) this.push(T.AMP_ASSIGN, "&=", l, c);
+		else this.push(T.AMP, "&", l, c);
+	}
+
+	_readOperator(ch, l, c) {
+		const punct = PUNCT_TOKEN[ch];
+		if (punct) {
+			this.push(punct[0], punct[1], l, c);
+			return;
+		}
+
+		const assign = ASSIGN_OP[ch];
+		if (assign) {
+			if (this.match("=")) this.push(assign[0], assign[1], l, c);
+			else this.push(assign[2], assign[3], l, c);
+			return;
+		}
+
+		switch (ch) {
+			case "+":
+				this._readPlusOp(l, c);
+				break;
+			case "-":
+				this._readMinusOp(l, c);
+				break;
+			case "<":
+				this._readLessOp(l, c);
+				break;
+			case ">":
+				this._readGreaterOp(l, c);
+				break;
+			case "&":
+				this._readAmpOp(l, c);
+				break;
+			case "|":
+				if (this.match("|")) this.push(T.OR, "||", l, c);
+				else if (this.match("=")) this.push(T.PIPE_ASSIGN, "|=", l, c);
+				else this.push(T.PIPE, "|", l, c);
+				break;
+			case ".":
+				if (this.peek() === "." && this.peek(1) === ".") {
+					this.advance();
+					this.advance();
+					this.push(T.ELLIPSIS, "...", l, c);
+				} else {
+					this.push(T.DOT, ".", l, c);
+				}
+				break;
+			default:
+				this.err(`Unexpected character: '${ch}'`);
+		}
+	}
 }