cluaupp 1.5.2 → 2.0.0-beta.1

package/src/compiler/lexer.ts

@@ -0,0 +1,444 @@
+/**
+ * CL++ 0.9 lexer (mirrors KartzRbx/CLPP `src/core/lexer/lexer.cpp`).
+ * `<<` line comments, `<<[ … ]>>` blocks, `<<[[]] … >>` doc comments, `<<!strict` directives,
+ * `.:` concat, `~>` connect, `::` scope, templates with `${}`, numbers with `_`, hex and binary.
+ */
+
+import type { DiagnosticBag } from "./diagnostics.js";
+import type { Span } from "./source.js";
+
+export type TokenKind =
+	| "ident"
+	| "keyword"
+	| "int"
+	| "float"
+	| "string"
+	| "template"
+	| "op"
+	| "directive"
+	| "doc"
+	| "eof";
+
+export interface TemplatePart {
+	/** Decoded literal text, or the source range of an `${ … }` hole. */
+	text?: string;
+	hole?: Span;
+}
+
+export interface Token {
+	kind: TokenKind;
+	/** Lexeme as written (decoded value lives in `value`). */
+	text: string;
+	start: number;
+	end: number;
+	/** Decoded string value, numeric value, or directive word. */
+	value?: string | number;
+	parts?: TemplatePart[];
+	/** First token on its line (used for statement-start recovery heuristics). */
+	newlineBefore: boolean;
+}
+
+export const KEYWORDS = new Set([
+	"Vector2", "Vector3", "Vector4", "abstract", "and", "as", "async", "atomic", "auto", "await", "bool", "break",
+	"buffer", "case", "catch", "const", "constexpr", "continue", "cout", "decltype", "default", "double", "else",
+	"endl", "enum", "extern", "false", "final", "float", "for", "from", "func", "guard", "if", "import", "in", "int",
+	"join", "let", "link", "match", "move", "mut", "namespace", "new", "not", "null", "observable", "operator", "or",
+	"override", "parallel", "pcall", "post", "private", "public", "report", "return", "shl", "shr", "signal", "spawn",
+	"static", "string", "struct", "switch", "task", "thread", "true", "try", "type", "using", "variant", "void",
+	"warn", "where", "while",
+]);
+
+/** Keywords that also name a value in Roblox code (`task.wait`, `string.format`, `buffer.create`, `type(x)`). */
+export const VALUE_KEYWORDS = new Set(["task", "string", "buffer", "type", "Vector2", "Vector3", "warn", "thread"]);
+
+const OPERATORS = [
+	"...", "<=", ">=", "==", "!=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "->", "=>", "~>", "::", ".:",
+	"..", "+", "-", "*", "/", "%", "=", "<", ">", "!", "&", "|", "^", "~", "?", ":", ".", ",", ";", "(", ")", "{",
+	"}", "[", "]", "@", "#",
+];
+
+function isIdentStart(c: number): boolean {
+	return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || c >= 0x80;
+}
+
+function isIdentPart(c: number): boolean {
+	return isIdentStart(c) || (c >= 48 && c <= 57);
+}
+
+function isDigit(c: number): boolean {
+	return c >= 48 && c <= 57;
+}
+
+const ESCAPES: Record<string, string> = {
+	n: "\n",
+	t: "\t",
+	r: "\r",
+	"0": "\0",
+	"\\": "\\",
+	'"': '"',
+	"'": "'",
+	"`": "`",
+	$: "$",
+	"{": "{",
+	"}": "}",
+};
+
+export class Lexer {
+	private pos = 0;
+	private readonly tokens: Token[] = [];
+	private newline = true;
+	private readonly text: string;
+
+	constructor(
+		text: string,
+		private readonly diagnostics: DiagnosticBag,
+		private readonly base = 0,
+		private readonly end = text.length,
+	) {
+		this.text = text;
+		this.pos = base;
+		if (base === 0 && text.charCodeAt(0) === 0xfeff) {
+			this.pos = 1;
+		}
+	}
+
+	tokenize(): Token[] {
+		while (true) {
+			this.skipTrivia();
+			if (this.pos >= this.end) {
+				this.push("eof", this.pos, this.pos);
+				return this.tokens;
+			}
+			this.next();
+		}
+	}
+
+	private push(kind: TokenKind, start: number, end: number, value?: string | number, parts?: TemplatePart[]): Token {
+		const token: Token = { kind, text: this.text.slice(start, end), start, end, newlineBefore: this.newline };
+		if (value !== undefined) {
+			token.value = value;
+		}
+		if (parts) {
+			token.parts = parts;
+		}
+		this.newline = false;
+		this.tokens.push(token);
+		return token;
+	}
+
+	private span(start: number, end: number): Span {
+		return { start, end };
+	}
+
+	private skipTrivia(): void {
+		const t = this.text;
+		while (this.pos < this.end) {
+			const c = t.charCodeAt(this.pos);
+			if (c === 10 || c === 13) {
+				this.newline = true;
+				this.pos++;
+				continue;
+			}
+			if (c === 32 || c === 9 || c === 11 || c === 12) {
+				this.pos++;
+				continue;
+			}
+			if (c === 60 && t.charCodeAt(this.pos + 1) === 60) {
+				// `<<` comment family
+				const start = this.pos;
+				const third = t[this.pos + 2];
+				if (third === "!") {
+					let p = this.pos + 3;
+					while (p < this.end && /[A-Za-z0-9_-]/.test(t[p])) {
+						p++;
+					}
+					const word = t.slice(this.pos + 3, p);
+					this.push("directive", start, p, word);
+					this.pos = p;
+					this.skipLine();
+					continue;
+				}
+				if (third === "[" && t[this.pos + 3] === "[" && t[this.pos + 4] === "]" && t[this.pos + 5] === "]") {
+					const close = t.indexOf(">>", this.pos + 6);
+					if (close < 0 || close >= this.end) {
+						this.diagnostics.error("CL1003", this.span(start, Math.min(this.end, start + 6)), "unterminated doc comment", ["close it with `>>`"]);
+						this.pos = this.end;
+						return;
+					}
+					this.push("doc", start, close + 2, t.slice(this.pos + 6, close).trim());
+					this.pos = close + 2;
+					continue;
+				}
+				if (third === "[") {
+					const close = t.indexOf("]>>", this.pos + 3);
+					if (close < 0 || close >= this.end) {
+						this.diagnostics.error("CL1002", this.span(start, Math.min(this.end, start + 3)), "unterminated block comment", ["close it with `]>>`"]);
+						this.pos = this.end;
+						return;
+					}
+					for (let i = this.pos; i < close; i++) {
+						const cc = t.charCodeAt(i);
+						if (cc === 10 || cc === 13) {
+							this.newline = true;
+						}
+					}
+					this.pos = close + 3;
+					continue;
+				}
+				this.skipLine();
+				continue;
+			}
+			break;
+		}
+	}
+
+	private skipLine(): void {
+		while (this.pos < this.end) {
+			const c = this.text.charCodeAt(this.pos);
+			if (c === 10 || c === 13) {
+				return;
+			}
+			this.pos++;
+		}
+	}
+
+	private next(): void {
+		const t = this.text;
+		const start = this.pos;
+		const c = t.charCodeAt(start);
+
+		if (isIdentStart(c)) {
+			let p = start + 1;
+			while (p < this.end && isIdentPart(t.charCodeAt(p))) {
+				p++;
+			}
+			this.pos = p;
+			const word = t.slice(start, p);
+			this.push(KEYWORDS.has(word) ? "keyword" : "ident", start, p);
+			return;
+		}
+		if (isDigit(c)) {
+			this.number(start);
+			return;
+		}
+		if (c === 34 || c === 39) {
+			this.string(start, c);
+			return;
+		}
+		if (c === 96) {
+			this.template(start);
+			return;
+		}
+		for (const op of OPERATORS) {
+			if (t.startsWith(op, start)) {
+				this.pos = start + op.length;
+				this.push("op", start, this.pos);
+				return;
+			}
+		}
+		this.pos = start + 1;
+		this.diagnostics.error("CL1001", this.span(start, this.pos), `unexpected character '${t[start]}'`);
+	}
+
+	private number(start: number): void {
+		const t = this.text;
+		let p = start;
+		let isFloat = false;
+		let bad = false;
+		const digits = (accept: (ch: string) => boolean) => {
+			let last = "";
+			let count = 0;
+			while (p < this.end) {
+				const ch = t[p];
+				if (ch === "_") {
+					if (!accept(last) || !accept(t[p + 1] ?? "")) {
+						bad = true;
+					}
+					last = ch;
+					p++;
+					continue;
+				}
+				if (!accept(ch)) {
+					break;
+				}
+				last = ch;
+				count++;
+				p++;
+			}
+			return count;
+		};
+		const dec = (ch: string) => ch >= "0" && ch <= "9";
+		if (t[p] === "0" && (t[p + 1] === "x" || t[p + 1] === "X")) {
+			p += 2;
+			const n = digits((ch) => /[0-9a-fA-F]/.test(ch));
+			if (n === 0) {
+				bad = true;
+			}
+		} else if (t[p] === "0" && (t[p + 1] === "b" || t[p + 1] === "B")) {
+			p += 2;
+			const n = digits((ch) => ch === "0" || ch === "1");
+			if (n === 0) {
+				bad = true;
+			}
+		} else {
+			digits(dec);
+			// `0..4` is a range, `3.5` a float, `3.` followed by a name is member access.
+			if (t[p] === "." && dec(t[p + 1] ?? "")) {
+				isFloat = true;
+				p++;
+				digits(dec);
+			}
+			if (t[p] === "e" || t[p] === "E") {
+				let q = p + 1;
+				if (t[q] === "+" || t[q] === "-") {
+					q++;
+				}
+				if (dec(t[q] ?? "")) {
+					isFloat = true;
+					p = q;
+					digits(dec);
+				}
+			}
+		}
+		this.pos = p;
+		const raw = t.slice(start, p);
+		const clean = raw.replace(/_/g, "");
+		let value: number;
+		if (/^0[xX]/.test(clean)) {
+			value = parseInt(clean.slice(2), 16);
+		} else if (/^0[bB]/.test(clean)) {
+			value = parseInt(clean.slice(2), 2);
+		} else {
+			value = Number(clean);
+		}
+		if (bad || !Number.isFinite(value)) {
+			this.diagnostics.error("CL1004", this.span(start, p), raw.includes("_") ? "invalid numeric separator" : "invalid numeric literal");
+			value = 0;
+		}
+		this.push(isFloat ? "float" : "int", start, p, value);
+	}
+
+	private escape(p: number, out: string[]): number {
+		const t = this.text;
+		const ch = t[p + 1];
+		if (ch === "u" && t[p + 2] === "{") {
+			const close = t.indexOf("}", p + 3);
+			const hex = close > 0 ? t.slice(p + 3, close) : "";
+			if (/^[0-9a-fA-F]{1,6}$/.test(hex)) {
+				out.push(String.fromCodePoint(parseInt(hex, 16)));
+				return close + 1;
+			}
+		}
+		const mapped = ch !== undefined ? ESCAPES[ch] : undefined;
+		if (mapped === undefined) {
+			this.diagnostics.error("CL1005", this.span(p, Math.min(this.end, p + 2)), `invalid escape '\\${ch ?? ""}'`, ["valid escapes: \\n \\t \\r \\0 \\\\ \\\" \\' \\` \\$ \\u{hex}"]);
+			out.push(ch ?? "");
+			return p + 2;
+		}
+		out.push(mapped);
+		return p + 2;
+	}
+
+	private string(start: number, quote: number): void {
+		const t = this.text;
+		let p = start + 1;
+		const out: string[] = [];
+		while (p < this.end) {
+			const c = t.charCodeAt(p);
+			if (c === quote) {
+				this.pos = p + 1;
+				this.push("string", start, this.pos, out.join(""));
+				return;
+			}
+			if (c === 10 || c === 13) {
+				break;
+			}
+			if (c === 92) {
+				p = this.escape(p, out);
+				continue;
+			}
+			out.push(t[p]);
+			p++;
+		}
+		this.pos = p;
+		this.diagnostics.error("CL1006", this.span(start, p), "unterminated string");
+		this.push("string", start, p, out.join(""));
+	}
+
+	private template(start: number): void {
+		const t = this.text;
+		let p = start + 1;
+		const parts: TemplatePart[] = [];
+		let literal: string[] = [];
+		const flush = () => {
+			if (literal.length) {
+				parts.push({ text: literal.join("") });
+				literal = [];
+			}
+		};
+		while (p < this.end) {
+			const c = t.charCodeAt(p);
+			if (c === 96) {
+				flush();
+				this.pos = p + 1;
+				this.push("template", start, this.pos, undefined, parts);
+				return;
+			}
+			if (c === 10 || c === 13) {
+				break;
+			}
+			if (c === 92) {
+				p = this.escape(p, literal);
+				continue;
+			}
+			if (c === 36 && t[p + 1] === "{") {
+				flush();
+				let depth = 1;
+				let q = p + 2;
+				let quote = 0;
+				while (q < this.end && depth > 0) {
+					const d = t.charCodeAt(q);
+					if (quote) {
+						if (d === 92) {
+							q += 2;
+							continue;
+						}
+						if (d === quote) {
+							quote = 0;
+						}
+					} else if (d === 34 || d === 39) {
+						quote = d;
+					} else if (d === 123) {
+						depth++;
+					} else if (d === 125) {
+						depth--;
+						if (depth === 0) {
+							break;
+						}
+					} else if (d === 10 || d === 13 || d === 96) {
+						break;
+					}
+					q++;
+				}
+				if (depth !== 0 || q >= this.end || t.charCodeAt(q) !== 125) {
+					this.diagnostics.error("CL1007", this.span(p, Math.min(this.end, q)), "unterminated interpolation", ["close `${` with `}`"]);
+					p = q;
+					break;
+				}
+				parts.push({ hole: { start: p + 2, end: q } });
+				p = q + 1;
+				continue;
+			}
+			literal.push(t[p]);
+			p++;
+		}
+		flush();
+		this.pos = p;
+		this.diagnostics.error("CL1008", this.span(start, p), "unterminated template string");
+		this.push("template", start, p, undefined, parts);
+	}
+}
+
+export function tokenize(text: string, diagnostics: DiagnosticBag, base = 0, end = text.length): Token[] {
+	return new Lexer(text, diagnostics, base, end).tokenize();
+}