@aexol/spectral 0.9.221 → 0.9.223

package/dist/generated/parser/lexer.js

@@ -0,0 +1,877 @@
+// GENERATED by cli-node/scripts/vendor-parser.mjs — do not edit; regenerate with `node scripts/vendor-parser.mjs`; source: packages/parser/
+/**
+ * Aexol Language Lexer
+ *
+ * Tokenizes Aexol source code into a stream of tokens.
+ * Handles keywords, identifiers, operators, strings, and whitespace.
+ */
+export var TokenType;
+(function (TokenType) {
+    // Literals
+    TokenType["IDENTIFIER"] = "IDENTIFIER";
+    TokenType["STRING"] = "STRING";
+    TokenType["NUMBER"] = "NUMBER";
+    // Keywords
+    TokenType["VISITOR"] = "VISITOR";
+    TokenType["AND"] = "AND";
+    TokenType["OR"] = "OR";
+    TokenType["NOT"] = "NOT";
+    TokenType["IF"] = "IF";
+    TokenType["WHEN"] = "WHEN";
+    TokenType["WHILE"] = "WHILE";
+    TokenType["FOR"] = "FOR";
+    TokenType["IN"] = "IN";
+    TokenType["OF"] = "OF";
+    TokenType["AS"] = "AS";
+    TokenType["WITH"] = "WITH";
+    TokenType["WITHOUT"] = "WITHOUT";
+    TokenType["REQUIRE"] = "REQUIRE";
+    TokenType["IMPORT"] = "IMPORT";
+    TokenType["FROM"] = "FROM";
+    TokenType["DEFINE"] = "DEFINE";
+    TokenType["TYPE"] = "TYPE";
+    TokenType["ENUM"] = "ENUM";
+    TokenType["INTERFACE"] = "INTERFACE";
+    TokenType["AGENT"] = "AGENT";
+    TokenType["WORKFLOW"] = "WORKFLOW";
+    TokenType["ROLE"] = "ROLE";
+    TokenType["PERMISSION"] = "PERMISSION";
+    TokenType["ACTION"] = "ACTION";
+    TokenType["STATE"] = "STATE";
+    TokenType["TRANSITION"] = "TRANSITION";
+    // Execution Plan Keywords
+    TokenType["EXECUTION_PLAN"] = "EXECUTION_PLAN";
+    TokenType["TASK"] = "TASK";
+    TokenType["SUBTASK"] = "SUBTASK";
+    TokenType["SUBTASKS"] = "SUBTASKS";
+    TokenType["DEPENDENCIES"] = "DEPENDENCIES";
+    TokenType["PARALLEL_WITH"] = "PARALLEL_WITH";
+    TokenType["EXECUTION_GRAPH"] = "EXECUTION_GRAPH";
+    TokenType["PHASE"] = "PHASE";
+    TokenType["CONTEXT"] = "CONTEXT";
+    // Agent/Task Properties
+    TokenType["CAPABILITIES"] = "CAPABILITIES";
+    TokenType["SKILLS"] = "SKILLS";
+    TokenType["MODEL"] = "MODEL";
+    TokenType["MAX_CONCURRENT"] = "MAX_CONCURRENT";
+    TokenType["ESTIMATED_DURATION"] = "ESTIMATED_DURATION";
+    TokenType["DESCRIPTION"] = "DESCRIPTION";
+    TokenType["SPEC_REFERENCE"] = "SPEC_REFERENCE";
+    TokenType["GENERATED_FROM"] = "GENERATED_FROM";
+    TokenType["GENERATED_AT"] = "GENERATED_AT";
+    TokenType["GENERATED_BY"] = "GENERATED_BY";
+    TokenType["INITIAL"] = "INITIAL";
+    TokenType["FINAL"] = "FINAL";
+    TokenType["TRUE"] = "TRUE";
+    TokenType["FALSE"] = "FALSE";
+    TokenType["NULL"] = "NULL";
+    TokenType["RETURN"] = "RETURN";
+    TokenType["YIELD"] = "YIELD";
+    TokenType["ASYNC"] = "ASYNC";
+    TokenType["AWAIT"] = "AWAIT";
+    TokenType["THROW"] = "THROW";
+    TokenType["TRY"] = "TRY";
+    TokenType["CATCH"] = "CATCH";
+    TokenType["FINALLY"] = "FINALLY";
+    // Operators
+    TokenType["LPAREN"] = "LPAREN";
+    TokenType["RPAREN"] = "RPAREN";
+    TokenType["LBRACE"] = "LBRACE";
+    TokenType["RBRACE"] = "RBRACE";
+    TokenType["LBRACKET"] = "LBRACKET";
+    TokenType["RBRACKET"] = "RBRACKET";
+    TokenType["DOT"] = "DOT";
+    TokenType["COMMA"] = "COMMA";
+    TokenType["COLON"] = "COLON";
+    TokenType["SEMICOLON"] = "SEMICOLON";
+    TokenType["ARROW"] = "ARROW";
+    TokenType["FAT_ARROW"] = "FAT_ARROW";
+    TokenType["EQUALS"] = "EQUALS";
+    TokenType["PLUS_EQUALS"] = "PLUS_EQUALS";
+    TokenType["MINUS_EQUALS"] = "MINUS_EQUALS";
+    TokenType["MULT_EQUALS"] = "MULT_EQUALS";
+    TokenType["DIV_EQUALS"] = "DIV_EQUALS";
+    TokenType["PLUS"] = "PLUS";
+    TokenType["MINUS"] = "MINUS";
+    TokenType["MULT"] = "MULT";
+    TokenType["DIV"] = "DIV";
+    TokenType["MOD"] = "MOD";
+    TokenType["POWER"] = "POWER";
+    TokenType["EQ"] = "EQ";
+    TokenType["NEQ"] = "NEQ";
+    TokenType["LT"] = "LT";
+    TokenType["LTE"] = "LTE";
+    TokenType["GT"] = "GT";
+    TokenType["GTE"] = "GTE";
+    TokenType["AMPERSAND"] = "AMPERSAND";
+    TokenType["PIPE"] = "PIPE";
+    TokenType["CARET"] = "CARET";
+    TokenType["TILDE"] = "TILDE";
+    TokenType["QUESTION"] = "QUESTION";
+    TokenType["EXCLAMATION"] = "EXCLAMATION";
+    TokenType["AT"] = "AT";
+    TokenType["DOLLAR"] = "DOLLAR";
+    TokenType["HASH"] = "HASH";
+    TokenType["BACKSLASH"] = "BACKSLASH";
+    // Special
+    TokenType["NEWLINE"] = "NEWLINE";
+    TokenType["INDENT"] = "INDENT";
+    TokenType["DEDENT"] = "DEDENT";
+    TokenType["EOF"] = "EOF";
+    TokenType["COMMENT"] = "COMMENT";
+    TokenType["WHITESPACE"] = "WHITESPACE";
+    TokenType["PROMPT"] = "PROMPT";
+})(TokenType || (TokenType = {}));
+const KEYWORDS = {
+    visitor: TokenType.VISITOR,
+    and: TokenType.AND,
+    or: TokenType.OR,
+    not: TokenType.NOT,
+    if: TokenType.IF,
+    when: TokenType.WHEN,
+    while: TokenType.WHILE,
+    for: TokenType.FOR,
+    in: TokenType.IN,
+    of: TokenType.OF,
+    as: TokenType.AS,
+    with: TokenType.WITH,
+    without: TokenType.WITHOUT,
+    require: TokenType.REQUIRE,
+    import: TokenType.IMPORT,
+    from: TokenType.FROM,
+    define: TokenType.DEFINE,
+    type: TokenType.TYPE,
+    enum: TokenType.ENUM,
+    interface: TokenType.INTERFACE,
+    agent: TokenType.AGENT,
+    workflow: TokenType.WORKFLOW,
+    role: TokenType.ROLE,
+    permission: TokenType.PERMISSION,
+    action: TokenType.ACTION,
+    state: TokenType.STATE,
+    transition: TokenType.TRANSITION,
+    // Execution Plan Keywords
+    execution_plan: TokenType.EXECUTION_PLAN,
+    task: TokenType.TASK,
+    subtask: TokenType.SUBTASK,
+    subtasks: TokenType.SUBTASKS,
+    dependencies: TokenType.DEPENDENCIES,
+    parallel_with: TokenType.PARALLEL_WITH,
+    execution_graph: TokenType.EXECUTION_GRAPH,
+    phase: TokenType.PHASE,
+    context: TokenType.CONTEXT,
+    // Agent/Task Properties
+    capabilities: TokenType.CAPABILITIES,
+    skills: TokenType.SKILLS,
+    model: TokenType.MODEL,
+    max_concurrent: TokenType.MAX_CONCURRENT,
+    estimated_duration: TokenType.ESTIMATED_DURATION,
+    description: TokenType.DESCRIPTION,
+    spec_reference: TokenType.SPEC_REFERENCE,
+    generated_from: TokenType.GENERATED_FROM,
+    generated_at: TokenType.GENERATED_AT,
+    generated_by: TokenType.GENERATED_BY,
+    initial: TokenType.INITIAL,
+    final: TokenType.FINAL,
+    true: TokenType.TRUE,
+    false: TokenType.FALSE,
+    null: TokenType.NULL,
+    return: TokenType.RETURN,
+    yield: TokenType.YIELD,
+    async: TokenType.ASYNC,
+    await: TokenType.AWAIT,
+    throw: TokenType.THROW,
+    try: TokenType.TRY,
+    catch: TokenType.CATCH,
+    finally: TokenType.FINALLY,
+};
+export class Lexer {
+    source;
+    position = 0;
+    line = 1;
+    column = 1;
+    tokens = [];
+    indentStack = [0];
+    constructor(source) {
+        this.source = source;
+    }
+    peek(offset = 0) {
+        const pos = this.position + offset;
+        return pos < this.source.length ? this.source[pos] : "\0";
+    }
+    advance() {
+        const char = this.source[this.position];
+        this.position++;
+        if (char === "\n") {
+            this.line++;
+            this.column = 1;
+        }
+        else {
+            this.column++;
+        }
+        return char;
+    }
+    skipWhitespace() {
+        while (this.peek() === " " || this.peek() === "\t" || this.peek() === "\r") {
+            this.advance();
+        }
+    }
+    skipCommentIfNeeded() {
+        const char = this.peek();
+        // Shell-style comments starting with '#'
+        if (char === "#") {
+            const start = this.position;
+            const startLine = this.line;
+            const startColumn = this.column;
+            while (this.peek() !== "\n" && this.peek() !== "\0") {
+                this.advance();
+            }
+            const value = this.source.substring(start, this.position);
+            this.tokens.push({
+                type: TokenType.COMMENT,
+                value,
+                location: { line: startLine, column: startColumn, offset: start },
+                raw: value,
+            });
+            return true;
+        }
+        // Line comments starting with '//' (common in plans)
+        if (char === "/" && this.peek(1) === "/") {
+            const start = this.position;
+            const startLine = this.line;
+            const startColumn = this.column;
+            // Skip the leading '//'
+            this.advance();
+            this.advance();
+            while (this.peek() !== "\n" && this.peek() !== "\0") {
+                this.advance();
+            }
+            const value = this.source.substring(start, this.position);
+            this.tokens.push({
+                type: TokenType.COMMENT,
+                value,
+                location: { line: startLine, column: startColumn, offset: start },
+                raw: value,
+            });
+            return true;
+        }
+        // Block comments starting with '/*' and ending with '*/'
+        if (char === "/" && this.peek(1) === "*") {
+            const start = this.position;
+            const startLine = this.line;
+            const startColumn = this.column;
+            // Skip the leading '/*'
+            this.advance();
+            this.advance();
+            while (this.peek() !== "\0" && !(this.peek() === "*" && this.peek(1) === "/")) {
+                this.advance();
+            }
+            // Consume the closing '*/' if present
+            if (this.peek() === "*" && this.peek(1) === "/") {
+                this.advance();
+                this.advance();
+            }
+            const value = this.source.substring(start, this.position);
+            this.tokens.push({
+                type: TokenType.COMMENT,
+                value,
+                location: { line: startLine, column: startColumn, offset: start },
+                raw: value,
+            });
+            return true;
+        }
+        return false;
+    }
+    readString(quote) {
+        const start = this.position;
+        const startLine = this.line;
+        const startColumn = this.column;
+        this.advance(); // Skip opening quote
+        let value = "";
+        while (this.peek() !== quote && this.peek() !== "\0") {
+            if (this.peek() === "\\") {
+                this.advance();
+                const escaped = this.peek();
+                switch (escaped) {
+                    case "n":
+                        value += "\n";
+                        break;
+                    case "t":
+                        value += "\t";
+                        break;
+                    case "r":
+                        value += "\r";
+                        break;
+                    case "\\":
+                        value += "\\";
+                        break;
+                    case '"':
+                        value += '"';
+                        break;
+                    case "'":
+                        value += "'";
+                        break;
+                    default:
+                        value += escaped;
+                }
+                this.advance();
+            }
+            else {
+                value += this.advance();
+            }
+        }
+        if (this.peek() === quote) {
+            this.advance(); // Skip closing quote
+        }
+        return {
+            type: TokenType.STRING,
+            value,
+            location: { line: startLine, column: startColumn, offset: start },
+            raw: this.source.substring(start, this.position),
+        };
+    }
+    readNumber() {
+        const start = this.position;
+        const startLine = this.line;
+        const startColumn = this.column;
+        while (/[0-9]/.test(this.peek())) {
+            this.advance();
+        }
+        if (this.peek() === "." && /[0-9]/.test(this.peek(1))) {
+            this.advance(); // Skip dot
+            while (/[0-9]/.test(this.peek())) {
+                this.advance();
+            }
+        }
+        const raw = this.source.substring(start, this.position);
+        return {
+            type: TokenType.NUMBER,
+            value: raw,
+            location: { line: startLine, column: startColumn, offset: start },
+            raw,
+        };
+    }
+    readIdentifier() {
+        const start = this.position;
+        const startLine = this.line;
+        const startColumn = this.column;
+        while (/[a-zA-Z0-9_-]/.test(this.peek())) {
+            this.advance();
+        }
+        const raw = this.source.substring(start, this.position);
+        const lower = raw.toLowerCase();
+        const type = KEYWORDS[lower] || TokenType.IDENTIFIER;
+        return {
+            type,
+            value: raw,
+            location: { line: startLine, column: startColumn, offset: start },
+            raw,
+        };
+    }
+    readPromptString() {
+        const start = this.position;
+        const startLine = this.line;
+        const startColumn = this.column;
+        // Skip opening '''
+        this.advance();
+        this.advance();
+        this.advance();
+        let value = "";
+        while (this.position < this.source.length) {
+            if (this.peek() === "'" && this.peek(1) === "'" && this.peek(2) === "'") {
+                this.advance();
+                this.advance();
+                this.advance();
+                break;
+            }
+            value += this.advance();
+        }
+        // Trim leading/trailing whitespace
+        value = value.trim();
+        return {
+            type: TokenType.PROMPT,
+            value,
+            location: { line: startLine, column: startColumn, offset: start },
+            raw: this.source.substring(start, this.position),
+        };
+    }
+    handleIndentation() {
+        const lineStart = this.position;
+        let spaces = 0;
+        while (this.peek() === " " || this.peek() === "\t") {
+            spaces += this.peek() === "\t" ? 4 : 1;
+            this.advance();
+        }
+        // Skip empty lines and comments
+        if (this.peek() === "\n" || this.peek() === "#" || this.peek() === "\0") {
+            return;
+        }
+        const currentIndent = this.indentStack[this.indentStack.length - 1];
+        if (spaces > currentIndent) {
+            this.indentStack.push(spaces);
+            this.tokens.push({
+                type: TokenType.INDENT,
+                value: " ".repeat(spaces),
+                location: { line: this.line, column: 1, offset: lineStart },
+                raw: " ".repeat(spaces),
+            });
+        }
+        else if (spaces < currentIndent) {
+            while (this.indentStack.length > 1 &&
+                this.indentStack[this.indentStack.length - 1] > spaces) {
+                this.indentStack.pop();
+                this.tokens.push({
+                    type: TokenType.DEDENT,
+                    value: "",
+                    location: { line: this.line, column: 1, offset: lineStart },
+                    raw: "",
+                });
+            }
+        }
+    }
+    tokenize() {
+        this.tokens = [];
+        while (this.position < this.source.length) {
+            const char = this.peek();
+            // Handle newline
+            if (char === "\n") {
+                const start = this.position;
+                const startLine = this.line;
+                const startColumn = this.column;
+                this.advance();
+                // Check if next line has content
+                let peekPos = this.position;
+                while (peekPos < this.source.length &&
+                    (this.source[peekPos] === " " || this.source[peekPos] === "\t")) {
+                    peekPos++;
+                }
+                // Only emit newline if not followed by EOF or only whitespace
+                if (peekPos < this.source.length && this.source[peekPos] !== "\n" &&
+                    this.source[peekPos] !== "#") {
+                    this.tokens.push({
+                        type: TokenType.NEWLINE,
+                        value: "\n",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "\n",
+                    });
+                    this.handleIndentation();
+                }
+                continue;
+            }
+            // Handle comments (supports '#', '//', and '/* */')
+            if (this.skipCommentIfNeeded()) {
+                continue;
+            }
+            // Skip whitespace (spaces, tabs)
+            if (char === " " || char === "\t") {
+                this.skipWhitespace();
+                continue;
+            }
+            // Handle triple single-quote strings (descriptions/prompts)
+            if (char === "'" && this.peek(1) === "'" && this.peek(2) === "'") {
+                this.tokens.push(this.readPromptString());
+                continue;
+            }
+            // Handle strings
+            if (char === '"' || char === "'") {
+                this.tokens.push(this.readString(char));
+                continue;
+            }
+            // Handle numbers
+            if (/[0-9]/.test(char)) {
+                this.tokens.push(this.readNumber());
+                continue;
+            }
+            // Handle identifiers and keywords
+            if (/[a-zA-Z_]/.test(char)) {
+                this.tokens.push(this.readIdentifier());
+                continue;
+            }
+            // Handle operators and punctuation
+            const start = this.position;
+            const startLine = this.line;
+            const startColumn = this.column;
+            if (char === "(" && this.peek(1) !== ")") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.LPAREN,
+                    value: "(",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "(",
+                });
+            }
+            else if (char === ")") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.RPAREN,
+                    value: ")",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ")",
+                });
+            }
+            else if (char === "{") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.LBRACE,
+                    value: "{",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "{",
+                });
+            }
+            else if (char === "}") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.RBRACE,
+                    value: "}",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "}",
+                });
+            }
+            else if (char === "[") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.LBRACKET,
+                    value: "[",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "[",
+                });
+            }
+            else if (char === "]") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.RBRACKET,
+                    value: "]",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "]",
+                });
+            }
+            else if (char === "." && !/[0-9]/.test(this.peek(1))) {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.DOT,
+                    value: ".",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ".",
+                });
+            }
+            else if (char === ",") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.COMMA,
+                    value: ",",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ",",
+                });
+            }
+            else if (char === ":" && this.peek(1) !== "=") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.COLON,
+                    value: ":",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ":",
+                });
+            }
+            else if (char === ";") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.SEMICOLON,
+                    value: ";",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ";",
+                });
+            }
+            else if (char === "-" && this.peek(1) === ">") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.ARROW,
+                    value: "->",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "->",
+                });
+            }
+            else if (char === "=" && this.peek(1) === ">") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.FAT_ARROW,
+                    value: "=>",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "=>",
+                });
+            }
+            else if (char === "=" && this.peek(1) === "=") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.EQ,
+                    value: "==",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "==",
+                });
+            }
+            else if (char === "!" && this.peek(1) === "=") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.NEQ,
+                    value: "!=",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "!=",
+                });
+            }
+            else if (char === "<" && this.peek(1) === "=") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.LTE,
+                    value: "<=",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "<=",
+                });
+            }
+            else if (char === ">" && this.peek(1) === "=") {
+                this.advance();
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.GTE,
+                    value: ">=",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ">=",
+                });
+            }
+            else if (char === "+") {
+                this.advance();
+                if (this.peek() === "=") {
+                    this.advance();
+                    this.tokens.push({
+                        type: TokenType.PLUS_EQUALS,
+                        value: "+=",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "+=",
+                    });
+                }
+                else {
+                    this.tokens.push({
+                        type: TokenType.PLUS,
+                        value: "+",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "+",
+                    });
+                }
+            }
+            else if (char === "-") {
+                this.advance();
+                if (this.peek() === "=") {
+                    this.advance();
+                    this.tokens.push({
+                        type: TokenType.MINUS_EQUALS,
+                        value: "-=",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "-=",
+                    });
+                }
+                else {
+                    this.tokens.push({
+                        type: TokenType.MINUS,
+                        value: "-",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "-",
+                    });
+                }
+            }
+            else if (char === "*") {
+                this.advance();
+                if (this.peek() === "*") {
+                    this.advance();
+                    this.tokens.push({
+                        type: TokenType.POWER,
+                        value: "**",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "**",
+                    });
+                }
+                else if (this.peek() === "=") {
+                    this.advance();
+                    this.tokens.push({
+                        type: TokenType.MULT_EQUALS,
+                        value: "*=",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "*=",
+                    });
+                }
+                else {
+                    this.tokens.push({
+                        type: TokenType.MULT,
+                        value: "*",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "*",
+                    });
+                }
+            }
+            else if (char === "/") {
+                this.advance();
+                if (this.peek() === "=") {
+                    this.advance();
+                    this.tokens.push({
+                        type: TokenType.DIV_EQUALS,
+                        value: "/=",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "/=",
+                    });
+                }
+                else {
+                    this.tokens.push({
+                        type: TokenType.DIV,
+                        value: "/",
+                        location: { line: startLine, column: startColumn, offset: start },
+                        raw: "/",
+                    });
+                }
+            }
+            else if (char === "%") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.MOD,
+                    value: "%",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "%",
+                });
+            }
+            else if (char === "=") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.EQUALS,
+                    value: "=",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "=",
+                });
+            }
+            else if (char === "<") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.LT,
+                    value: "<",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "<",
+                });
+            }
+            else if (char === ">") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.GT,
+                    value: ">",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: ">",
+                });
+            }
+            else if (char === "&") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.AMPERSAND,
+                    value: "&",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "&",
+                });
+            }
+            else if (char === "|") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.PIPE,
+                    value: "|",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "|",
+                });
+            }
+            else if (char === "^") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.CARET,
+                    value: "^",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "^",
+                });
+            }
+            else if (char === "~") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.TILDE,
+                    value: "~",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "~",
+                });
+            }
+            else if (char === "?") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.QUESTION,
+                    value: "?",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "?",
+                });
+            }
+            else if (char === "!") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.EXCLAMATION,
+                    value: "!",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "!",
+                });
+            }
+            else if (char === "@") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.AT,
+                    value: "@",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "@",
+                });
+            }
+            else if (char === "$") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.DOLLAR,
+                    value: "$",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "$",
+                });
+            }
+            else if (char === "\\") {
+                this.advance();
+                this.tokens.push({
+                    type: TokenType.BACKSLASH,
+                    value: "\\",
+                    location: { line: startLine, column: startColumn, offset: start },
+                    raw: "\\",
+                });
+            }
+            else {
+                // Unknown character, skip it
+                this.advance();
+            }
+        }
+        // Handle remaining dedents
+        while (this.indentStack.length > 1) {
+            this.indentStack.pop();
+            this.tokens.push({
+                type: TokenType.DEDENT,
+                value: "",
+                location: { line: this.line, column: 1, offset: this.position },
+                raw: "",
+            });
+        }
+        // Add EOF token
+        this.tokens.push({
+            type: TokenType.EOF,
+            value: "",
+            location: { line: this.line, column: this.column, offset: this.position },
+            raw: "",
+        });
+        return this.tokens;
+    }
+    getTokens() {
+        return this.tokens;
+    }
+}