@quereus/quereus 0.1.0

package/dist/src/parser/lexer.js

@@ -0,0 +1,762 @@
+export var TokenType;
+(function (TokenType) {
+    // Literals
+    TokenType["INTEGER"] = "INTEGER";
+    TokenType["FLOAT"] = "FLOAT";
+    TokenType["STRING"] = "STRING";
+    TokenType["IDENTIFIER"] = "IDENTIFIER";
+    TokenType["BLOB"] = "BLOB";
+    // Keywords
+    TokenType["SELECT"] = "SELECT";
+    TokenType["FROM"] = "FROM";
+    TokenType["WHERE"] = "WHERE";
+    TokenType["INSERT"] = "INSERT";
+    TokenType["UPDATE"] = "UPDATE";
+    TokenType["DELETE"] = "DELETE";
+    TokenType["CREATE"] = "CREATE";
+    TokenType["DROP"] = "DROP";
+    TokenType["ALTER"] = "ALTER";
+    TokenType["TABLE"] = "TABLE";
+    TokenType["INDEX"] = "INDEX";
+    TokenType["VIEW"] = "VIEW";
+    TokenType["TEMP"] = "TEMP";
+    TokenType["TEMPORARY"] = "TEMPORARY";
+    TokenType["VIRTUAL"] = "VIRTUAL";
+    TokenType["USING"] = "USING";
+    TokenType["INTO"] = "INTO";
+    TokenType["NULL"] = "NULL";
+    TokenType["TRUE"] = "TRUE";
+    TokenType["FALSE"] = "FALSE";
+    TokenType["NOT"] = "NOT";
+    TokenType["AND"] = "AND";
+    TokenType["OR"] = "OR";
+    TokenType["IN"] = "IN";
+    TokenType["LIKE"] = "LIKE";
+    TokenType["BETWEEN"] = "BETWEEN";
+    TokenType["IS"] = "IS";
+    TokenType["AS"] = "AS";
+    TokenType["DISTINCT"] = "DISTINCT";
+    TokenType["GROUP"] = "GROUP";
+    TokenType["BY"] = "BY";
+    TokenType["HAVING"] = "HAVING";
+    TokenType["ORDER"] = "ORDER";
+    TokenType["ASC"] = "ASC";
+    TokenType["DESC"] = "DESC";
+    TokenType["LIMIT"] = "LIMIT";
+    TokenType["OFFSET"] = "OFFSET";
+    TokenType["UNION"] = "UNION";
+    TokenType["INTERSECT"] = "INTERSECT";
+    TokenType["EXCEPT"] = "EXCEPT";
+    TokenType["ALL"] = "ALL";
+    TokenType["PRIMARY"] = "PRIMARY";
+    TokenType["CONSTRAINT"] = "CONSTRAINT";
+    TokenType["GENERATED"] = "GENERATED";
+    TokenType["COLLATE"] = "COLLATE";
+    TokenType["KEY"] = "KEY";
+    TokenType["UNIQUE"] = "UNIQUE";
+    TokenType["DEFAULT"] = "DEFAULT";
+    TokenType["CHECK"] = "CHECK";
+    TokenType["FOREIGN"] = "FOREIGN";
+    TokenType["REFERENCES"] = "REFERENCES";
+    TokenType["AUTOINCREMENT"] = "AUTOINCREMENT";
+    TokenType["ON"] = "ON";
+    TokenType["CONFLICT"] = "CONFLICT";
+    TokenType["CASCADE"] = "CASCADE";
+    TokenType["RESTRICT"] = "RESTRICT";
+    TokenType["SET"] = "SET";
+    TokenType["NO"] = "NO";
+    TokenType["ACTION"] = "ACTION";
+    TokenType["RENAME"] = "RENAME";
+    TokenType["COLUMN"] = "COLUMN";
+    TokenType["TO"] = "TO";
+    TokenType["ADD"] = "ADD";
+    TokenType["ALWAYS"] = "ALWAYS";
+    TokenType["ABORT"] = "ABORT";
+    TokenType["FAIL"] = "FAIL";
+    TokenType["IGNORE"] = "IGNORE";
+    TokenType["BEGIN"] = "BEGIN";
+    TokenType["COMMIT"] = "COMMIT";
+    TokenType["ROLLBACK"] = "ROLLBACK";
+    TokenType["TRANSACTION"] = "TRANSACTION";
+    TokenType["DEFERRED"] = "DEFERRED";
+    TokenType["IMMEDIATE"] = "IMMEDIATE";
+    TokenType["EXCLUSIVE"] = "EXCLUSIVE";
+    TokenType["JOIN"] = "JOIN";
+    TokenType["INNER"] = "INNER";
+    TokenType["LEFT"] = "LEFT";
+    TokenType["RIGHT"] = "RIGHT";
+    TokenType["FULL"] = "FULL";
+    TokenType["CROSS"] = "CROSS";
+    TokenType["OUTER"] = "OUTER";
+    TokenType["NATURAL"] = "NATURAL";
+    TokenType["REPLACE"] = "REPLACE";
+    TokenType["VALUES"] = "VALUES";
+    TokenType["EXISTS"] = "EXISTS";
+    TokenType["IF"] = "IF";
+    TokenType["DEFERRABLE"] = "DEFERRABLE";
+    TokenType["INITIALLY"] = "INITIALLY";
+    TokenType["STORED"] = "STORED";
+    TokenType["RETURNING"] = "RETURNING";
+    TokenType["SAVEPOINT"] = "SAVEPOINT";
+    TokenType["RELEASE"] = "RELEASE";
+    TokenType["PRAGMA"] = "PRAGMA";
+    TokenType["WITH"] = "WITH";
+    TokenType["RECURSIVE"] = "RECURSIVE";
+    TokenType["XOR"] = "XOR";
+    TokenType["CASE"] = "CASE";
+    TokenType["WHEN"] = "WHEN";
+    TokenType["THEN"] = "THEN";
+    TokenType["ELSE"] = "ELSE";
+    TokenType["END"] = "END";
+    TokenType["CAST"] = "CAST";
+    TokenType["OVER"] = "OVER";
+    TokenType["PARTITION"] = "PARTITION";
+    TokenType["ROW"] = "ROW";
+    TokenType["ROWS"] = "ROWS";
+    TokenType["RANGE"] = "RANGE";
+    TokenType["UNBOUNDED"] = "UNBOUNDED";
+    TokenType["PRECEDING"] = "PRECEDING";
+    TokenType["FOLLOWING"] = "FOLLOWING";
+    TokenType["CURRENT"] = "CURRENT";
+    // Operators and punctuation
+    TokenType["PLUS"] = "PLUS";
+    TokenType["MINUS"] = "MINUS";
+    TokenType["ASTERISK"] = "ASTERISK";
+    TokenType["SLASH"] = "SLASH";
+    TokenType["PERCENT"] = "PERCENT";
+    TokenType["EQUAL"] = "EQUAL";
+    TokenType["EQUAL_EQUAL"] = "EQUAL_EQUAL";
+    TokenType["NOT_EQUAL"] = "NOT_EQUAL";
+    TokenType["LESS"] = "LESS";
+    TokenType["LESS_EQUAL"] = "LESS_EQUAL";
+    TokenType["GREATER"] = "GREATER";
+    TokenType["GREATER_EQUAL"] = "GREATER_EQUAL";
+    TokenType["LPAREN"] = "LPAREN";
+    TokenType["RPAREN"] = "RPAREN";
+    TokenType["COMMA"] = "COMMA";
+    TokenType["DOT"] = "DOT";
+    TokenType["SEMICOLON"] = "SEMICOLON";
+    TokenType["TILDE"] = "TILDE";
+    TokenType["PIPE"] = "PIPE";
+    TokenType["PIPE_PIPE"] = "PIPE_PIPE";
+    TokenType["AMPERSAND"] = "AMPERSAND";
+    TokenType["AMPERSAND_AMPERSAND"] = "AMPERSAND_AMPERSAND";
+    TokenType["QUESTION"] = "QUESTION";
+    TokenType["COLON"] = "COLON";
+    TokenType["DOLLAR"] = "DOLLAR";
+    TokenType["ARROW"] = "ARROW";
+    // Special
+    TokenType["EOF"] = "EOF";
+    TokenType["ERROR"] = "ERROR";
+})(TokenType || (TokenType = {}));
+// Reserved keywords mapping
+export const KEYWORDS = {
+    'select': TokenType.SELECT,
+    'from': TokenType.FROM,
+    'where': TokenType.WHERE,
+    'insert': TokenType.INSERT,
+    'update': TokenType.UPDATE,
+    'delete': TokenType.DELETE,
+    'create': TokenType.CREATE,
+    'drop': TokenType.DROP,
+    'alter': TokenType.ALTER,
+    'table': TokenType.TABLE,
+    'index': TokenType.INDEX,
+    'view': TokenType.VIEW,
+    'virtual': TokenType.VIRTUAL,
+    'using': TokenType.USING,
+    'null': TokenType.NULL,
+    'true': TokenType.TRUE,
+    'false': TokenType.FALSE,
+    'not': TokenType.NOT,
+    'and': TokenType.AND,
+    'or': TokenType.OR,
+    'in': TokenType.IN,
+    'like': TokenType.LIKE,
+    'between': TokenType.BETWEEN,
+    'is': TokenType.IS,
+    'as': TokenType.AS,
+    'distinct': TokenType.DISTINCT,
+    'group': TokenType.GROUP,
+    'by': TokenType.BY,
+    'having': TokenType.HAVING,
+    'order': TokenType.ORDER,
+    'asc': TokenType.ASC,
+    'desc': TokenType.DESC,
+    'limit': TokenType.LIMIT,
+    'offset': TokenType.OFFSET,
+    'union': TokenType.UNION,
+    'all': TokenType.ALL,
+    'primary': TokenType.PRIMARY,
+    'constraint': TokenType.CONSTRAINT,
+    'key': TokenType.KEY,
+    'unique': TokenType.UNIQUE,
+    'default': TokenType.DEFAULT,
+    'check': TokenType.CHECK,
+    'collate': TokenType.COLLATE,
+    'foreign': TokenType.FOREIGN,
+    'references': TokenType.REFERENCES,
+    'on': TokenType.ON,
+    'conflict': TokenType.CONFLICT,
+    'cascade': TokenType.CASCADE,
+    'restrict': TokenType.RESTRICT,
+    'set': TokenType.SET,
+    'autoincrement': TokenType.AUTOINCREMENT,
+    'no': TokenType.NO,
+    'action': TokenType.ACTION,
+    'begin': TokenType.BEGIN,
+    'commit': TokenType.COMMIT,
+    'rollback': TokenType.ROLLBACK,
+    'transaction': TokenType.TRANSACTION,
+    'deferred': TokenType.DEFERRED,
+    'immediate': TokenType.IMMEDIATE,
+    'exclusive': TokenType.EXCLUSIVE,
+    'deferrable': TokenType.DEFERRABLE,
+    'initially': TokenType.INITIALLY,
+    'stored': TokenType.STORED,
+    'returning': TokenType.RETURNING,
+    'join': TokenType.JOIN,
+    'inner': TokenType.INNER,
+    'left': TokenType.LEFT,
+    'right': TokenType.RIGHT,
+    'full': TokenType.FULL,
+    'cross': TokenType.CROSS,
+    'outer': TokenType.OUTER,
+    'natural': TokenType.NATURAL,
+    'replace': TokenType.REPLACE,
+    'values': TokenType.VALUES,
+    'exists': TokenType.EXISTS,
+    'if': TokenType.IF,
+    'into': TokenType.INTO,
+    'temp': TokenType.TEMP,
+    'temporary': TokenType.TEMPORARY,
+    'rename': TokenType.RENAME,
+    'to': TokenType.TO,
+    'add': TokenType.ADD,
+    'always': TokenType.ALWAYS,
+    'abort': TokenType.ABORT,
+    'fail': TokenType.FAIL,
+    'ignore': TokenType.IGNORE,
+    'savepoint': TokenType.SAVEPOINT,
+    'release': TokenType.RELEASE,
+    'pragma': TokenType.PRAGMA,
+    'with': TokenType.WITH,
+    'recursive': TokenType.RECURSIVE,
+    'xor': TokenType.XOR,
+    'case': TokenType.CASE,
+    'when': TokenType.WHEN,
+    'then': TokenType.THEN,
+    'else': TokenType.ELSE,
+    'end': TokenType.END,
+    'cast': TokenType.CAST,
+    'over': TokenType.OVER,
+    'partition': TokenType.PARTITION,
+    'row': TokenType.ROW,
+    'rows': TokenType.ROWS,
+    'range': TokenType.RANGE,
+    'unbounded': TokenType.UNBOUNDED,
+    'preceding': TokenType.PRECEDING,
+    'following': TokenType.FOLLOWING,
+    'current': TokenType.CURRENT,
+    'intersect': TokenType.INTERSECT,
+    'except': TokenType.EXCEPT,
+};
+/**
+ * Lexer class for tokenizing SQL statements
+ */
+export class Lexer {
+    source;
+    tokens = [];
+    start = 0;
+    current = 0;
+    line = 1;
+    column = 1;
+    startLine = 1;
+    startColumn = 1;
+    constructor(source) {
+        this.source = source;
+    }
+    /**
+     * Scans the input and returns all tokens.
+     */
+    scanTokens() {
+        while (!this.isAtEnd()) {
+            this.start = this.current;
+            this.startLine = this.line;
+            this.startColumn = this.column;
+            this.scanToken();
+        }
+        this.tokens.push({
+            type: TokenType.EOF,
+            lexeme: '',
+            startLine: this.line,
+            startColumn: this.column,
+            startOffset: this.source.length,
+            endLine: this.line,
+            endColumn: this.column,
+            endOffset: this.source.length,
+        });
+        return this.tokens;
+    }
+    isAtEnd() {
+        return this.current >= this.source.length;
+    }
+    scanToken() {
+        const c = this.advance();
+        switch (c) {
+            // Single-character tokens
+            case '(':
+                this.addToken(TokenType.LPAREN);
+                break;
+            case ')':
+                this.addToken(TokenType.RPAREN);
+                break;
+            case ',':
+                this.addToken(TokenType.COMMA);
+                break;
+            case '.':
+                this.addToken(TokenType.DOT);
+                break;
+            case ';':
+                this.addToken(TokenType.SEMICOLON);
+                break;
+            case '+':
+                this.addToken(TokenType.PLUS);
+                break;
+            case '-':
+                if (this.match('-')) {
+                    // SQL-style line comment
+                    while (this.peek() !== '\n' && !this.isAtEnd()) {
+                        this.advance();
+                    }
+                }
+                else if (this.match('>')) {
+                    this.addToken(TokenType.ARROW);
+                }
+                else {
+                    this.addToken(TokenType.MINUS);
+                }
+                break;
+            case '*':
+                this.addToken(TokenType.ASTERISK);
+                break;
+            case '/':
+                if (this.match('/')) {
+                    // Single line comment
+                    while (this.peek() !== '\n' && !this.isAtEnd()) {
+                        this.advance();
+                    }
+                }
+                else if (this.match('*')) {
+                    // Multiline comment
+                    this.multilineComment();
+                }
+                else {
+                    this.addToken(TokenType.SLASH);
+                }
+                break;
+            case '%':
+                this.addToken(TokenType.PERCENT);
+                break;
+            case '~':
+                this.addToken(TokenType.TILDE);
+                break;
+            case '?':
+                this.addToken(TokenType.QUESTION);
+                break;
+            case ':':
+                this.addToken(TokenType.COLON);
+                break;
+            case '$':
+                this.addToken(TokenType.DOLLAR);
+                break;
+            // One or two character tokens
+            case '=':
+                this.addToken(this.match('=') ? TokenType.EQUAL_EQUAL : TokenType.EQUAL);
+                break;
+            case '!':
+                this.addToken(this.match('=') ? TokenType.NOT_EQUAL : TokenType.ERROR);
+                break;
+            case '<':
+                if (this.match('=')) {
+                    this.addToken(TokenType.LESS_EQUAL);
+                }
+                else if (this.match('>')) {
+                    this.addToken(TokenType.NOT_EQUAL);
+                }
+                else {
+                    this.addToken(TokenType.LESS);
+                }
+                break;
+            case '>':
+                this.addToken(this.match('=') ? TokenType.GREATER_EQUAL : TokenType.GREATER);
+                break;
+            case '|':
+                this.addToken(this.match('|') ? TokenType.PIPE_PIPE : TokenType.PIPE);
+                break;
+            case '&':
+                this.addToken(this.match('&') ? TokenType.AMPERSAND_AMPERSAND : TokenType.AMPERSAND);
+                break;
+            // String literals
+            case '\'':
+                this.string('\'');
+                break;
+            case '"':
+                this.string('"');
+                break;
+            case '`':
+                this.backtickIdentifier();
+                break;
+            case '[':
+                this.bracketIdentifier();
+                break;
+            // Blob literals
+            case 'x':
+            case 'X':
+                if (this.match('\'')) {
+                    this.blobLiteral();
+                }
+                else {
+                    this.identifier();
+                }
+                break;
+            // Whitespace
+            case ' ':
+            case '\r':
+            case '\t':
+                // Ignore whitespace
+                break;
+            case '\n':
+                this.line++;
+                this.column = 1;
+                break;
+            // Default - handle identifiers and numbers
+            default:
+                if (this.isDigit(c)) {
+                    this.number();
+                }
+                else if (this.isAlpha(c)) {
+                    this.identifier();
+                }
+                else {
+                    this.addErrorToken(`Unexpected character: ${c}`);
+                }
+                break;
+        }
+    }
+    advance() {
+        const char = this.source.charAt(this.current);
+        this.current++;
+        if (char === '\n') {
+            this.line++;
+            this.column = 1;
+        }
+        else {
+            this.column++;
+        }
+        return char;
+    }
+    match(expected) {
+        if (this.isAtEnd())
+            return false;
+        if (this.source.charAt(this.current) !== expected)
+            return false;
+        this.current++;
+        this.column++;
+        return true;
+    }
+    peek() {
+        if (this.isAtEnd())
+            return '\0';
+        return this.source.charAt(this.current);
+    }
+    peekNext() {
+        if (this.current + 1 >= this.source.length)
+            return '\0';
+        return this.source.charAt(this.current + 1);
+    }
+    string(quote) {
+        let value = '';
+        let escaping = false;
+        while ((!this.isAtEnd() && this.peek() !== quote) || escaping) {
+            if (this.peek() === '\n') {
+                this.line++;
+                this.column = 1;
+            }
+            if (escaping) {
+                // Handle escape sequences
+                const c = this.peek();
+                switch (c) {
+                    case 'n':
+                        value += '\n';
+                        break;
+                    case 'r':
+                        value += '\r';
+                        break;
+                    case 't':
+                        value += '\t';
+                        break;
+                    case '\\':
+                        value += '\\';
+                        break;
+                    case '\'':
+                        value += '\'';
+                        break;
+                    case '"':
+                        value += '"';
+                        break;
+                    case '0':
+                        value += '\0';
+                        break;
+                    default:
+                        value += c;
+                        break;
+                }
+                escaping = false;
+            }
+            else if (this.peek() === '\\') {
+                escaping = true;
+            }
+            else {
+                value += this.peek();
+            }
+            this.advance();
+        }
+        if (this.isAtEnd()) {
+            this.addErrorToken("Unterminated string.");
+            return;
+        }
+        // Consume the closing quote
+        this.advance();
+        // SQLite allows adjacent string literals to be concatenated
+        if (this.peek() === quote) {
+            this.advance(); // Consume the opening quote of the next string
+            this.string(quote); // Process the next string
+            // Merge the two string tokens
+            if (this.tokens.length > 0 && this.tokens[this.tokens.length - 1].type === TokenType.STRING) {
+                const prevToken = this.tokens.pop();
+                value += prevToken.literal;
+            }
+        }
+        this.addToken(TokenType.STRING, value);
+    }
+    backtickIdentifier() {
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== '`') {
+            if (this.peek() === '\n') {
+                this.line++;
+                this.column = 1;
+            }
+            value += this.advance();
+        }
+        if (this.isAtEnd()) {
+            this.addErrorToken("Unterminated identifier.");
+            return;
+        }
+        // Consume the closing backtick
+        this.advance();
+        this.addToken(TokenType.IDENTIFIER, value);
+    }
+    bracketIdentifier() {
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== ']') {
+            if (this.peek() === '\n') {
+                this.line++;
+                this.column = 1;
+            }
+            value += this.advance();
+        }
+        if (this.isAtEnd()) {
+            this.addErrorToken("Unterminated identifier.");
+            return;
+        }
+        // Consume the closing bracket
+        this.advance();
+        this.addToken(TokenType.IDENTIFIER, value);
+    }
+    blobLiteral() {
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== '\'') {
+            if (this.isHexDigit(this.peek())) {
+                value += this.advance();
+            }
+            else if (this.isWhitespace(this.peek())) {
+                this.advance(); // Skip whitespace in blob literals
+            }
+            else {
+                this.addErrorToken("Invalid character in blob literal.");
+                return;
+            }
+        }
+        if (this.isAtEnd()) {
+            this.addErrorToken("Unterminated blob literal.");
+            return;
+        }
+        // Consume the closing quote
+        this.advance();
+        // Validate hex string length
+        if (value.length % 2 !== 0) {
+            this.addErrorToken("Blob literal must have an even number of hex digits.");
+            return;
+        }
+        // Convert hex string to Uint8Array
+        try {
+            const bytes = new Uint8Array(value.length / 2);
+            for (let i = 0; i < value.length; i += 2) {
+                bytes[i / 2] = parseInt(value.substring(i, i + 2), 16);
+            }
+            this.addToken(TokenType.BLOB, bytes);
+        }
+        catch {
+            this.addErrorToken("Invalid blob literal.");
+        }
+    }
+    number() {
+        let isFloat = false;
+        // Capture original lexeme starting from the first digit
+        const start = this.start; // Use the start offset saved before scanToken called number()
+        // Consume digits before decimal point
+        while (this.isDigit(this.peek())) {
+            this.advance();
+        }
+        // Check for decimal point
+        if (this.peek() === '.' && this.isDigit(this.peekNext())) {
+            isFloat = true;
+            this.advance(); // Consume the '.'
+            // Consume digits after decimal point
+            while (this.isDigit(this.peek())) {
+                this.advance();
+            }
+        }
+        // Check for exponent part
+        if (this.peek().toLowerCase() === 'e') {
+            isFloat = true;
+            this.advance(); // Consume the 'e' or 'E'
+            // Optional sign
+            if (this.peek() === '+' || this.peek() === '-') {
+                this.advance();
+            }
+            // Exponent digits
+            if (!this.isDigit(this.peek())) {
+                this.addErrorToken("Invalid number literal: expected digits after exponent.");
+                return;
+            }
+            while (this.isDigit(this.peek())) {
+                this.advance();
+            }
+        }
+        const lexeme = this.source.substring(start, this.current);
+        if (isFloat) {
+            // Store original string as literal for FLOAT
+            this.addToken(TokenType.FLOAT, lexeme);
+        }
+        else {
+            // For integers, parse now to handle potential BigInt
+            try {
+                const num = parseInt(lexeme, 10);
+                if (!Number.isSafeInteger(num)) {
+                    try {
+                        this.addToken(TokenType.INTEGER, BigInt(lexeme));
+                    }
+                    catch {
+                        this.addErrorToken("Integer literal too large.");
+                    }
+                }
+                else {
+                    this.addToken(TokenType.INTEGER, num);
+                }
+            }
+            catch {
+                try {
+                    this.addToken(TokenType.INTEGER, BigInt(lexeme));
+                }
+                catch {
+                    this.addErrorToken("Invalid integer literal.");
+                }
+            }
+        }
+    }
+    identifier() {
+        while (this.isAlphaNumeric(this.peek())) {
+            this.advance();
+        }
+        // Check if the identifier is a keyword
+        const text = this.source.substring(this.start, this.current).toLowerCase();
+        const type = KEYWORDS[text] || TokenType.IDENTIFIER;
+        this.addToken(type);
+    }
+    multilineComment() {
+        let nesting = 1; // Support nested comments
+        while (nesting > 0 && !this.isAtEnd()) {
+            if (this.peek() === '/' && this.peekNext() === '*') {
+                this.advance(); // Consume '/'
+                this.advance(); // Consume '*'
+                nesting++;
+            }
+            else if (this.peek() === '*' && this.peekNext() === '/') {
+                this.advance(); // Consume '*'
+                this.advance(); // Consume '/'
+                nesting--;
+            }
+            else {
+                if (this.peek() === '\n') {
+                    this.line++;
+                    this.column = 1;
+                }
+                else {
+                    this.column++;
+                }
+                this.advance();
+            }
+        }
+        if (nesting > 0) {
+            this.addErrorToken("Unterminated comment.");
+        }
+    }
+    isDigit(c) {
+        return c >= '0' && c <= '9';
+    }
+    isHexDigit(c) {
+        return (c >= '0' && c <= '9') ||
+            (c >= 'a' && c <= 'f') ||
+            (c >= 'A' && c <= 'F');
+    }
+    isAlpha(c) {
+        return (c >= 'a' && c <= 'z') ||
+            (c >= 'A' && c <= 'Z') ||
+            c === '_';
+    }
+    isAlphaNumeric(c) {
+        return this.isAlpha(c) || this.isDigit(c);
+    }
+    isWhitespace(c) {
+        return c === ' ' || c === '\r' || c === '\n' || c === '\t';
+    }
+    addToken(type, literal) {
+        const lexeme = this.source.substring(this.start, this.current);
+        this.tokens.push({
+            type,
+            lexeme, // Ensure lexeme is always the original string
+            literal,
+            startLine: this.startLine,
+            startColumn: this.startColumn,
+            startOffset: this.start,
+            endLine: this.line,
+            endColumn: this.column - 1,
+            endOffset: this.current,
+        });
+    }
+    addErrorToken(message) {
+        this.tokens.push({
+            type: TokenType.ERROR,
+            lexeme: message,
+            startLine: this.line,
+            startColumn: this.column - 1,
+            startOffset: this.current - 1,
+            endLine: this.line,
+            endColumn: this.column - 1,
+            endOffset: this.current,
+        });
+    }
+}
+//# sourceMappingURL=lexer.js.map