cluaupp 1.5.2 → 2.0.0-beta.2

package/generated/compiler/lexer.js

@@ -0,0 +1,403 @@
+"use strict";
+/**
+ * CL++ 0.9 lexer (mirrors KartzRbx/CLPP `src/core/lexer/lexer.cpp`).
+ * `<<` line comments, `<<[ … ]>>` blocks, `<<[[]] … >>` doc comments, `<<!strict` directives,
+ * `.:` concat, `~>` connect, `::` scope, templates with `${}`, numbers with `_`, hex and binary.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lexer = exports.VALUE_KEYWORDS = exports.KEYWORDS = void 0;
+exports.tokenize = tokenize;
+exports.KEYWORDS = new Set([
+    "Vector2", "Vector3", "Vector4", "abstract", "and", "as", "async", "atomic", "auto", "await", "bool", "break",
+    "buffer", "case", "catch", "const", "constexpr", "continue", "cout", "decltype", "default", "double", "else",
+    "endl", "enum", "extern", "false", "final", "float", "for", "from", "func", "guard", "if", "import", "in", "int",
+    "join", "let", "link", "match", "move", "mut", "namespace", "new", "not", "null", "observable", "operator", "or",
+    "override", "parallel", "pcall", "post", "private", "public", "report", "return", "shl", "shr", "signal", "spawn",
+    "static", "string", "struct", "switch", "task", "thread", "true", "try", "type", "using", "variant", "void",
+    "warn", "where", "while",
+]);
+/** Keywords that also name a value in Roblox code (`task.wait`, `string.format`, `buffer.create`, `type(x)`). */
+exports.VALUE_KEYWORDS = new Set(["task", "string", "buffer", "type", "Vector2", "Vector3", "warn", "thread"]);
+const OPERATORS = [
+    "...", "<=", ">=", "==", "!=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "->", "=>", "~>", "::", ".:",
+    "..", "+", "-", "*", "/", "%", "=", "<", ">", "!", "&", "|", "^", "~", "?", ":", ".", ",", ";", "(", ")", "{",
+    "}", "[", "]", "@", "#",
+];
+function isIdentStart(c) {
+    return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || c >= 0x80;
+}
+function isIdentPart(c) {
+    return isIdentStart(c) || (c >= 48 && c <= 57);
+}
+function isDigit(c) {
+    return c >= 48 && c <= 57;
+}
+const ESCAPES = {
+    n: "\n",
+    t: "\t",
+    r: "\r",
+    "0": "\0",
+    "\\": "\\",
+    '"': '"',
+    "'": "'",
+    "`": "`",
+    $: "$",
+    "{": "{",
+    "}": "}",
+};
+class Lexer {
+    diagnostics;
+    base;
+    end;
+    pos = 0;
+    tokens = [];
+    newline = true;
+    text;
+    constructor(text, diagnostics, base = 0, end = text.length) {
+        this.diagnostics = diagnostics;
+        this.base = base;
+        this.end = end;
+        this.text = text;
+        this.pos = base;
+        if (base === 0 && text.charCodeAt(0) === 0xfeff) {
+            this.pos = 1;
+        }
+    }
+    tokenize() {
+        while (true) {
+            this.skipTrivia();
+            if (this.pos >= this.end) {
+                this.push("eof", this.pos, this.pos);
+                return this.tokens;
+            }
+            this.next();
+        }
+    }
+    push(kind, start, end, value, parts) {
+        const token = { kind, text: this.text.slice(start, end), start, end, newlineBefore: this.newline };
+        if (value !== undefined) {
+            token.value = value;
+        }
+        if (parts) {
+            token.parts = parts;
+        }
+        this.newline = false;
+        this.tokens.push(token);
+        return token;
+    }
+    span(start, end) {
+        return { start, end };
+    }
+    skipTrivia() {
+        const t = this.text;
+        while (this.pos < this.end) {
+            const c = t.charCodeAt(this.pos);
+            if (c === 10 || c === 13) {
+                this.newline = true;
+                this.pos++;
+                continue;
+            }
+            if (c === 32 || c === 9 || c === 11 || c === 12) {
+                this.pos++;
+                continue;
+            }
+            if (c === 60 && t.charCodeAt(this.pos + 1) === 60) {
+                // `<<` comment family
+                const start = this.pos;
+                const third = t[this.pos + 2];
+                if (third === "!") {
+                    let p = this.pos + 3;
+                    while (p < this.end && /[A-Za-z0-9_-]/.test(t[p])) {
+                        p++;
+                    }
+                    const word = t.slice(this.pos + 3, p);
+                    this.push("directive", start, p, word);
+                    this.pos = p;
+                    this.skipLine();
+                    continue;
+                }
+                if (third === "[" && t[this.pos + 3] === "[" && t[this.pos + 4] === "]" && t[this.pos + 5] === "]") {
+                    const close = t.indexOf(">>", this.pos + 6);
+                    if (close < 0 || close >= this.end) {
+                        this.diagnostics.error("CL1003", this.span(start, Math.min(this.end, start + 6)), "unterminated doc comment", ["close it with `>>`"]);
+                        this.pos = this.end;
+                        return;
+                    }
+                    this.push("doc", start, close + 2, t.slice(this.pos + 6, close).trim());
+                    this.pos = close + 2;
+                    continue;
+                }
+                if (third === "[") {
+                    const close = t.indexOf("]>>", this.pos + 3);
+                    if (close < 0 || close >= this.end) {
+                        this.diagnostics.error("CL1002", this.span(start, Math.min(this.end, start + 3)), "unterminated block comment", ["close it with `]>>`"]);
+                        this.pos = this.end;
+                        return;
+                    }
+                    for (let i = this.pos; i < close; i++) {
+                        const cc = t.charCodeAt(i);
+                        if (cc === 10 || cc === 13) {
+                            this.newline = true;
+                        }
+                    }
+                    this.pos = close + 3;
+                    continue;
+                }
+                this.skipLine();
+                continue;
+            }
+            break;
+        }
+    }
+    skipLine() {
+        while (this.pos < this.end) {
+            const c = this.text.charCodeAt(this.pos);
+            if (c === 10 || c === 13) {
+                return;
+            }
+            this.pos++;
+        }
+    }
+    next() {
+        const t = this.text;
+        const start = this.pos;
+        const c = t.charCodeAt(start);
+        if (isIdentStart(c)) {
+            let p = start + 1;
+            while (p < this.end && isIdentPart(t.charCodeAt(p))) {
+                p++;
+            }
+            this.pos = p;
+            const word = t.slice(start, p);
+            this.push(exports.KEYWORDS.has(word) ? "keyword" : "ident", start, p);
+            return;
+        }
+        if (isDigit(c)) {
+            this.number(start);
+            return;
+        }
+        if (c === 34 || c === 39) {
+            this.string(start, c);
+            return;
+        }
+        if (c === 96) {
+            this.template(start);
+            return;
+        }
+        for (const op of OPERATORS) {
+            if (t.startsWith(op, start)) {
+                this.pos = start + op.length;
+                this.push("op", start, this.pos);
+                return;
+            }
+        }
+        this.pos = start + 1;
+        this.diagnostics.error("CL1001", this.span(start, this.pos), `unexpected character '${t[start]}'`);
+    }
+    number(start) {
+        const t = this.text;
+        let p = start;
+        let isFloat = false;
+        let bad = false;
+        const digits = (accept) => {
+            let last = "";
+            let count = 0;
+            while (p < this.end) {
+                const ch = t[p];
+                if (ch === "_") {
+                    if (!accept(last) || !accept(t[p + 1] ?? "")) {
+                        bad = true;
+                    }
+                    last = ch;
+                    p++;
+                    continue;
+                }
+                if (!accept(ch)) {
+                    break;
+                }
+                last = ch;
+                count++;
+                p++;
+            }
+            return count;
+        };
+        const dec = (ch) => ch >= "0" && ch <= "9";
+        if (t[p] === "0" && (t[p + 1] === "x" || t[p + 1] === "X")) {
+            p += 2;
+            const n = digits((ch) => /[0-9a-fA-F]/.test(ch));
+            if (n === 0) {
+                bad = true;
+            }
+        }
+        else if (t[p] === "0" && (t[p + 1] === "b" || t[p + 1] === "B")) {
+            p += 2;
+            const n = digits((ch) => ch === "0" || ch === "1");
+            if (n === 0) {
+                bad = true;
+            }
+        }
+        else {
+            digits(dec);
+            // `0..4` is a range, `3.5` a float, `3.` followed by a name is member access.
+            if (t[p] === "." && dec(t[p + 1] ?? "")) {
+                isFloat = true;
+                p++;
+                digits(dec);
+            }
+            if (t[p] === "e" || t[p] === "E") {
+                let q = p + 1;
+                if (t[q] === "+" || t[q] === "-") {
+                    q++;
+                }
+                if (dec(t[q] ?? "")) {
+                    isFloat = true;
+                    p = q;
+                    digits(dec);
+                }
+            }
+        }
+        this.pos = p;
+        const raw = t.slice(start, p);
+        const clean = raw.replace(/_/g, "");
+        let value;
+        if (/^0[xX]/.test(clean)) {
+            value = parseInt(clean.slice(2), 16);
+        }
+        else if (/^0[bB]/.test(clean)) {
+            value = parseInt(clean.slice(2), 2);
+        }
+        else {
+            value = Number(clean);
+        }
+        if (bad || !Number.isFinite(value)) {
+            this.diagnostics.error("CL1004", this.span(start, p), raw.includes("_") ? "invalid numeric separator" : "invalid numeric literal");
+            value = 0;
+        }
+        this.push(isFloat ? "float" : "int", start, p, value);
+    }
+    escape(p, out) {
+        const t = this.text;
+        const ch = t[p + 1];
+        if (ch === "u" && t[p + 2] === "{") {
+            const close = t.indexOf("}", p + 3);
+            const hex = close > 0 ? t.slice(p + 3, close) : "";
+            if (/^[0-9a-fA-F]{1,6}$/.test(hex)) {
+                out.push(String.fromCodePoint(parseInt(hex, 16)));
+                return close + 1;
+            }
+        }
+        const mapped = ch !== undefined ? ESCAPES[ch] : undefined;
+        if (mapped === undefined) {
+            this.diagnostics.error("CL1005", this.span(p, Math.min(this.end, p + 2)), `invalid escape '\\${ch ?? ""}'`, ["valid escapes: \\n \\t \\r \\0 \\\\ \\\" \\' \\` \\$ \\u{hex}"]);
+            out.push(ch ?? "");
+            return p + 2;
+        }
+        out.push(mapped);
+        return p + 2;
+    }
+    string(start, quote) {
+        const t = this.text;
+        let p = start + 1;
+        const out = [];
+        while (p < this.end) {
+            const c = t.charCodeAt(p);
+            if (c === quote) {
+                this.pos = p + 1;
+                this.push("string", start, this.pos, out.join(""));
+                return;
+            }
+            if (c === 10 || c === 13) {
+                break;
+            }
+            if (c === 92) {
+                p = this.escape(p, out);
+                continue;
+            }
+            out.push(t[p]);
+            p++;
+        }
+        this.pos = p;
+        this.diagnostics.error("CL1006", this.span(start, p), "unterminated string");
+        this.push("string", start, p, out.join(""));
+    }
+    template(start) {
+        const t = this.text;
+        let p = start + 1;
+        const parts = [];
+        let literal = [];
+        const flush = () => {
+            if (literal.length) {
+                parts.push({ text: literal.join("") });
+                literal = [];
+            }
+        };
+        while (p < this.end) {
+            const c = t.charCodeAt(p);
+            if (c === 96) {
+                flush();
+                this.pos = p + 1;
+                this.push("template", start, this.pos, undefined, parts);
+                return;
+            }
+            if (c === 10 || c === 13) {
+                break;
+            }
+            if (c === 92) {
+                p = this.escape(p, literal);
+                continue;
+            }
+            if (c === 36 && t[p + 1] === "{") {
+                flush();
+                let depth = 1;
+                let q = p + 2;
+                let quote = 0;
+                while (q < this.end && depth > 0) {
+                    const d = t.charCodeAt(q);
+                    if (quote) {
+                        if (d === 92) {
+                            q += 2;
+                            continue;
+                        }
+                        if (d === quote) {
+                            quote = 0;
+                        }
+                    }
+                    else if (d === 34 || d === 39) {
+                        quote = d;
+                    }
+                    else if (d === 123) {
+                        depth++;
+                    }
+                    else if (d === 125) {
+                        depth--;
+                        if (depth === 0) {
+                            break;
+                        }
+                    }
+                    else if (d === 10 || d === 13 || d === 96) {
+                        break;
+                    }
+                    q++;
+                }
+                if (depth !== 0 || q >= this.end || t.charCodeAt(q) !== 125) {
+                    this.diagnostics.error("CL1007", this.span(p, Math.min(this.end, q)), "unterminated interpolation", ["close `${` with `}`"]);
+                    p = q;
+                    break;
+                }
+                parts.push({ hole: { start: p + 2, end: q } });
+                p = q + 1;
+                continue;
+            }
+            literal.push(t[p]);
+            p++;
+        }
+        flush();
+        this.pos = p;
+        this.diagnostics.error("CL1008", this.span(start, p), "unterminated template string");
+        this.push("template", start, p, undefined, parts);
+    }
+}
+exports.Lexer = Lexer;
+function tokenize(text, diagnostics, base = 0, end = text.length) {
+    return new Lexer(text, diagnostics, base, end).tokenize();
+}