flowquery 1.0.0 → 1.0.2

package/dist/tokenization/tokenizer.js

@@ -0,0 +1,220 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const keyword_1 = __importDefault(require("./keyword"));
+const token_1 = __importDefault(require("./token"));
+const string_walker_1 = __importDefault(require("./string_walker"));
+const string_utils_1 = __importDefault(require("../utils/string_utils"));
+const symbol_1 = __importDefault(require("./symbol"));
+const operator_1 = __importDefault(require("./operator"));
+const token_mapper_1 = __importDefault(require("./token_mapper"));
+/**
+ * Tokenizes FlowQuery input strings into a sequence of tokens.
+ *
+ * The tokenizer performs lexical analysis, breaking down the input text into
+ * meaningful tokens such as keywords, identifiers, operators, strings, numbers,
+ * and symbols. It handles comments, whitespace, and f-strings.
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = new Tokenizer("WITH x = 1 RETURN x");
+ * const tokens = tokenizer.tokenize();
+ * ```
+ */
+class Tokenizer {
+    /**
+     * Creates a new Tokenizer instance for the given input.
+     *
+     * @param input - The FlowQuery input string to tokenize
+     */
+    constructor(input) {
+        this.keywords = new token_mapper_1.default(keyword_1.default);
+        this.symbols = new token_mapper_1.default(symbol_1.default);
+        this.operators = new token_mapper_1.default(operator_1.default);
+        this.walker = new string_walker_1.default(input);
+    }
+    /**
+     * Tokenizes the input string into an array of tokens.
+     *
+     * @returns An array of Token objects representing the tokenized input
+     * @throws {Error} If an unrecognized token is encountered
+     */
+    tokenize() {
+        const tokens = [];
+        let last = null;
+        while (!this.walker.isAtEnd) {
+            tokens.push(...this.f_string());
+            last = this.getLastNonWhitespaceOrNonCommentToken(tokens) || last;
+            const token = this.getNextToken(last);
+            if (token === null) {
+                throw new Error(`Unrecognized token at position ${this.walker.position}`);
+            }
+            token.position = this.walker.position;
+            tokens.push(token);
+        }
+        return tokens;
+    }
+    getLastNonWhitespaceOrNonCommentToken(tokens) {
+        if (tokens.length === 0) {
+            return null;
+        }
+        if (!tokens[tokens.length - 1].isWhitespaceOrComment()) {
+            return tokens[tokens.length - 1];
+        }
+        return null;
+    }
+    getNextToken(last = null) {
+        if (this.walker.isAtEnd) {
+            return token_1.default.EOF;
+        }
+        return (this.comment() ||
+            this.whitespace() ||
+            this.lookup(this.keywords) ||
+            this.lookup(this.operators, last, this.skipMinus) ||
+            this.identifier() ||
+            this.string() ||
+            this.number() ||
+            this.lookup(this.symbols));
+    }
+    comment() {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForSingleComment() || this.walker.checkForMultiLineComment()) {
+            const uncommented = string_utils_1.default.uncomment(this.walker.getString(startPosition));
+            return token_1.default.COMMENT(uncommented);
+        }
+        return null;
+    }
+    identifier() {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForUnderScore() || this.walker.checkForLetter()) {
+            while (!this.walker.isAtEnd && (this.walker.checkForLetter() || this.walker.checkForDigit() || this.walker.checkForUnderScore())) {
+                ;
+            }
+            return token_1.default.IDENTIFIER(this.walker.getString(startPosition));
+        }
+        return null;
+    }
+    string() {
+        const startPosition = this.walker.position;
+        const quoteChar = this.walker.checkForQuote();
+        if (quoteChar === null) {
+            return null;
+        }
+        while (!this.walker.isAtEnd) {
+            if (this.walker.escaped(quoteChar)) {
+                this.walker.moveNext();
+                this.walker.moveNext();
+                continue;
+            }
+            if (this.walker.checkForString(quoteChar)) {
+                const value = this.walker.getString(startPosition);
+                if (quoteChar === symbol_1.default.BACKTICK) {
+                    return token_1.default.BACKTICK_STRING(value, quoteChar);
+                }
+                return token_1.default.STRING(value, quoteChar);
+            }
+            this.walker.moveNext();
+        }
+        throw new Error(`Unterminated string at position ${startPosition}`);
+    }
+    *f_string() {
+        if (!this.walker.checkForFStringStart()) {
+            return;
+        }
+        this.walker.moveNext(); // skip the f
+        let position = this.walker.position;
+        const quoteChar = this.walker.checkForQuote();
+        if (quoteChar === null) {
+            return;
+        }
+        while (!this.walker.isAtEnd) {
+            if (this.walker.escaped(quoteChar) || this.walker.escapedBrace()) {
+                this.walker.moveNext();
+                this.walker.moveNext();
+                continue;
+            }
+            if (this.walker.openingBrace()) {
+                yield token_1.default.F_STRING(this.walker.getString(position), quoteChar);
+                position = this.walker.position;
+                yield token_1.default.OPENING_BRACE;
+                this.walker.moveNext(); // skip the opening brace
+                position = this.walker.position;
+                while (!this.walker.isAtEnd && !this.walker.closingBrace()) {
+                    const token = this.getNextToken();
+                    if (token !== null) {
+                        yield token;
+                    }
+                    else {
+                        break;
+                    }
+                    if (this.walker.closingBrace()) {
+                        yield token_1.default.CLOSING_BRACE;
+                        this.walker.moveNext(); // skip the closing brace
+                        position = this.walker.position;
+                        break;
+                    }
+                }
+            }
+            if (this.walker.checkForString(quoteChar)) {
+                yield token_1.default.F_STRING(this.walker.getString(position), quoteChar);
+                return;
+            }
+            ;
+            this.walker.moveNext();
+        }
+    }
+    whitespace() {
+        let foundWhitespace = false;
+        while (!this.walker.isAtEnd && this.walker.checkForWhitespace()) {
+            this.walker.moveNext();
+            foundWhitespace = true;
+        }
+        return foundWhitespace ? token_1.default.WHITESPACE : null;
+    }
+    number() {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForString('-') || this.walker.checkForDigit()) {
+            while (!this.walker.isAtEnd && this.walker.checkForDigit()) {
+                ;
+            }
+            if (this.walker.checkForString(symbol_1.default.DOT)) {
+                while (!this.walker.isAtEnd && this.walker.checkForDigit()) {
+                    ;
+                }
+            }
+            const _number = this.walker.getString(startPosition);
+            return token_1.default.NUMBER(_number);
+        }
+        return null;
+    }
+    lookup(mapper, last = null, skip) {
+        const token = mapper.map(this.walker.getRemainingString());
+        if (token !== undefined && token.value !== null) {
+            if (token.can_be_identifier && this.walker.word_continuation(token.value)) {
+                return null;
+            }
+            if (skip && last && skip(last, token)) {
+                return null;
+            }
+            this.walker.moveBy(token.value.length);
+            if (mapper.last_found !== null) {
+                token.case_sensitive_value = mapper.last_found;
+            }
+            return token;
+        }
+        return null;
+    }
+    skipMinus(last, current) {
+        if (last === null) {
+            return false;
+        }
+        if ((last.isKeyword() || last.isComma() || last.isColon()) && current.isNegation()) {
+            return true;
+        }
+        return false;
+    }
+}
+exports.default = Tokenizer;
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenization/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/tokenization/tokenizer.ts"],"names":[],"mappings":";;;;;AAAA,wDAAgC;AAChC,oDAA4B;AAC5B,oEAA2C;AAC3C,yEAAgD;AAChD,sDAA8B;AAC9B,0DAAkC;AAClC,kEAAyC;AAEzC;;;;;;;;;;;;GAYG;AACH,MAAM,SAAS;IAMX;;;;OAIG;IACH,YAAY,KAAa;QATjB,aAAQ,GAAgB,IAAI,sBAAW,CAAC,iBAAO,CAAC,CAAC;QACjD,YAAO,GAAgB,IAAI,sBAAW,CAAC,gBAAM,CAAC,CAAC;QAC/C,cAAS,GAAgB,IAAI,sBAAW,CAAC,kBAAQ,CAAC,CAAC;QAQvD,IAAI,CAAC,MAAM,GAAG,IAAI,uBAAY,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;OAKG;IACI,QAAQ;QACX,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAiB,IAAI,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAChC,IAAI,GAAG,IAAI,CAAC,qCAAqC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9E,CAAC;YACD,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,qCAAqC,CAAC,MAAe;QACzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACpD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,OAAqB,IAAI;QAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,eAAK,CAAC,GAAG,CAAC;QACrB,CAAC;QACD,OAAO,CACH,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,UAAU,EAAE;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACjD,IAAI,CAAC,UAAU,EAAE;YACjB,IAAI,CAAC,MAAM,EAAE;YACb,IAAI,CAAC,MAAM,EAAE;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAC5B,CAAC;IACN,CAAC;IAEM,OAAO;QACV,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE,EAAE,CAAC;YAChF,MAAM,WAAW,GAAG,sBAAW,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC;YAChF,OAAO,eAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,UAAU;QACd,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,EAAE,CAAC;gBAC/H,CAAC;YACL,CAAC;YACD,OAAO,eAAK,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM;QACV,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QAC9C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACvB,SAAS;YACb,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBACnD,IAAI,SAAS,KAAK,gBAAM,CAAC,QAAQ,EAAE,CAAC;oBAChC,OAAO,eAAK,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBACnD,CAAC;gBACD,OAAO,eAAK,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC3B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mCAAmC,aAAa,EAAE,CAAC,CAAC;IACxE,CAAC;IAEO,CAAC,QAAQ;QACb,IAAG,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,aAAa;QACrC,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QAC9C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC/D,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACvB,SAAS;YACb,CAAC;YACD,IAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5B,MAAM,eAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;gBACjE,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChC,MAAM,eAAK,CAAC,aAAa,CAAC;gBAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,yBAAyB;gBACjD,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChC,OAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;oBACxD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClC,IAAG,KAAK,KAAK,IAAI,EAAE,CAAC;wBAChB,MAAM,KAAK,CAAC;oBAChB,CAAC;yBAAM,CAAC;wBACJ,MAAM;oBACV,CAAC;oBACD,IAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5B,MAAM,eAAK,CAAC,aAAa,CAAC;wBAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,yBAAyB;wBACjD,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;wBAChC,MAAM;oBACV,CAAC;gBACL,CAAC;YACL,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxC,MAAM,eAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;gBACjE,OAAO;YACX,CAAC;YAAA,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC3B,CAAC;IACL,CAAC;IAEO,UAAU;QACd,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvB,eAAe,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,OAAO,eAAe,CAAC,CAAC,CAAC,eAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,CAAC;IAEO,MAAM;QACV,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC;gBACzD,CAAC;YACL,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,gBAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC;oBACzD,CAAC;gBACL,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACrD,OAAO,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,MAAmB,EAAE,OAAqB,IAAI,EAAE,IAAsD;QACjH,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC9C,IAAG,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvE,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACpC,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACvC,IAAG,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;gBAC5B,KAAK,CAAC,oBAAoB,GAAG,MAAM,CAAC,UAAU,CAAC;YACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAkB,EAAE,OAAc;QAChD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;YAChF,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ;AAED,kBAAe,SAAS,CAAC"}

package/dist/tokenization/trie.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Represents a node in a Trie data structure.
+ *
+ * Each node can have children nodes (one per character) and may contain a token
+ * if the path to this node represents a complete word.
+ */
+class Node {
+    constructor() {
+        this._children = new Map();
+        this._token = undefined;
+    }
+    map(char) {
+        return this._children.get(char) || this._children.set(char, new Node()).get(char);
+    }
+    retrieve(char) {
+        return this._children.get(char);
+    }
+    set token(token) {
+        this._token = token;
+    }
+    get token() {
+        return this._token;
+    }
+    is_end_of_word() {
+        return this._token !== undefined;
+    }
+    no_children() {
+        return this._children.size === 0;
+    }
+}
+/**
+ * Trie (prefix tree) data structure for efficient keyword and operator lookup.
+ *
+ * Used during tokenization to quickly match input strings against known keywords
+ * and operators. Supports case-insensitive matching and tracks the longest match found.
+ *
+ * @example
+ * ```typescript
+ * const trie = new Trie();
+ * trie.insert(Token.WITH);
+ * const found = trie.find("WITH");
+ * ```
+ */
+class Trie {
+    constructor() {
+        this._root = new Node();
+        this._max_length = 0;
+        this._last_found = null;
+    }
+    /**
+     * Inserts a token into the trie.
+     *
+     * @param token - The token to insert
+     * @throws {Error} If the token value is null or empty
+     */
+    insert(token) {
+        if (token.value === null || token.value.length === 0) {
+            throw new Error("Token value cannot be null or empty");
+        }
+        let currentNode = this._root;
+        for (const char of token.value) {
+            currentNode = currentNode.map(char.toLowerCase());
+        }
+        if (token.value.length > this._max_length) {
+            this._max_length = token.value.length;
+        }
+        currentNode.token = token;
+    }
+    /**
+     * Finds a token by searching for the longest matching prefix in the trie.
+     *
+     * @param value - The string value to search for
+     * @returns The token if found, undefined otherwise
+     */
+    find(value) {
+        if (value.length === 0) {
+            return undefined;
+        }
+        let index = 0;
+        let current = undefined;
+        let found = undefined;
+        this._last_found = null;
+        while ((current = (current || this._root).retrieve(value[index].toLowerCase())) !== undefined) {
+            if (current.is_end_of_word()) {
+                found = current.token;
+                this._last_found = value.substring(0, index + 1);
+            }
+            index++;
+            if (index === value.length || index > this._max_length) {
+                break;
+            }
+        }
+        if (current === null || current === void 0 ? void 0 : current.is_end_of_word()) {
+            found = current.token;
+            this._last_found = value.substring(0, index);
+        }
+        return found;
+    }
+    /**
+     * Gets the last matched string from the most recent find operation.
+     *
+     * @returns The last found string, or null if no match was found
+     */
+    get last_found() {
+        return this._last_found;
+    }
+}
+exports.default = Trie;
+//# sourceMappingURL=trie.js.map

package/dist/tokenization/trie.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trie.js","sourceRoot":"","sources":["../../src/tokenization/trie.ts"],"names":[],"mappings":";;AAEA;;;;;GAKG;AACH,MAAM,IAAI;IAAV;QACY,cAAS,GAAsB,IAAI,GAAG,EAAE,CAAC;QACzC,WAAM,GAAsB,SAAS,CAAC;IAyBlD,CAAC;IAvBU,GAAG,CAAC,IAAY;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;IACvF,CAAC;IAEM,QAAQ,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,KAAK,CAAC,KAAY;QACzB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC;IACrC,CAAC;IAEM,WAAW;QACd,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC;IACrC,CAAC;CACJ;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,IAAI;IAAV;QACY,UAAK,GAAS,IAAI,IAAI,EAAE,CAAC;QACzB,gBAAW,GAAW,CAAC,CAAC;QACxB,gBAAW,GAAkB,IAAI,CAAC;IA6D9C,CAAC;IA3DG;;;;;OAKG;IACI,MAAM,CAAC,KAAY;QACtB,IAAG,KAAK,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC7B,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAC1C,CAAC;QACD,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACI,IAAI,CAAC,KAAa;QACrB,IAAG,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAqB,SAAS,CAAC;QAC1C,IAAI,KAAK,GAAsB,SAAS,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,OAAM,CAAC,OAAO,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3F,IAAG,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1B,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;gBACtB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACrD,CAAC;YACD,KAAK,EAAE,CAAC;YACR,IAAG,KAAK,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpD,MAAM;YACV,CAAC;QACL,CAAC;QACD,IAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,cAAc,EAAE,EAAE,CAAC;YAC3B,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;CACJ;AAED,kBAAe,IAAI,CAAC"}

package/dist/utils/object_utils.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Utility class for object-related operations.
+ */
+class ObjectUtils {
+    /**
+     * Checks if an object is an instance of any of the provided classes.
+     *
+     * @param obj - The object to check
+     * @param classes - Array of class constructors to test against
+     * @returns True if the object is an instance of any class, false otherwise
+     */
+    static isInstanceOfAny(obj, classes) {
+        return classes.some(cls => obj instanceof cls);
+    }
+}
+exports.default = ObjectUtils;
+//# sourceMappingURL=object_utils.js.map

package/dist/utils/object_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"object_utils.js","sourceRoot":"","sources":["../../src/utils/object_utils.ts"],"names":[],"mappings":";;AAAA;;GAEG;AACH,MAAM,WAAW;IACb;;;;;;OAMG;IACH,MAAM,CAAC,eAAe,CAAC,GAAQ,EAAE,OAAc;QAC3C,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,kBAAe,WAAW,CAAC"}

package/dist/utils/string_utils.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Utility class for string manipulation and validation.
+ *
+ * Provides methods for handling quoted strings, comments, escape sequences,
+ * and identifier validation.
+ */
+class StringUtils {
+    /**
+     * Removes surrounding quotes from a string.
+     *
+     * @param str - The string to unquote
+     * @returns The unquoted string
+     */
+    static unquote(str) {
+        if (str.length === 0) {
+            return str;
+        }
+        if (str.length === 1 && StringUtils.quotes.includes(str)) {
+            return '';
+        }
+        const first = str[0];
+        const last = str[str.length - 1];
+        if (StringUtils.quotes.includes(first) && first === last) {
+            return str.substring(1, str.length - 1);
+        }
+        if (StringUtils.quotes.includes(last) && first !== last) {
+            return str.substring(0, str.length - 1);
+        }
+        if (StringUtils.quotes.includes(first) && first !== last) {
+            return str.substring(1);
+        }
+        return str;
+    }
+    /**
+     * Removes comment markers from a string.
+     *
+     * @param str - The comment string
+     * @returns The string without comment markers
+     */
+    static uncomment(str) {
+        if (str.length < 2) {
+            return str;
+        }
+        if (str[0] === '/' && str[1] === '/') {
+            return str.substring(2);
+        }
+        if (str[0] === '/' && str[1] === '*' && str[str.length - 2] === '*' && str[str.length - 1] === '/') {
+            return str.substring(2, str.length - 2);
+        }
+        return str;
+    }
+    /**
+     * Removes escape sequences before quotes in a string.
+     *
+     * @param str - The string to process
+     * @param quoteChar - The quote character that was escaped
+     * @returns The string with escape sequences removed
+     */
+    static removeEscapedQuotes(str, quoteChar) {
+        let unescaped = '';
+        for (let i = 0; i < str.length; i++) {
+            if (str[i] === '\\' && str[i + 1] === quoteChar) {
+                i++;
+            }
+            unescaped += str[i];
+        }
+        return unescaped;
+    }
+    /**
+     * Removes escaped braces ({{ and }}) from f-strings.
+     *
+     * @param str - The string to process
+     * @returns The string with escaped braces resolved
+     */
+    static removeEscapedBraces(str) {
+        let unescaped = '';
+        for (let i = 0; i < str.length; i++) {
+            if ((str[i] === '{' && str[i + 1] === '{') || (str[i] === '}' && str[i + 1] === '}')) {
+                i++;
+            }
+            unescaped += str[i];
+        }
+        return unescaped;
+    }
+    /**
+     * Checks if a string is a valid identifier.
+     *
+     * @param str - The string to validate
+     * @returns True if the string can be used as an identifier, false otherwise
+     */
+    static can_be_identifier(str) {
+        const lower = str.toLowerCase();
+        if (lower.length === 0) {
+            return false;
+        }
+        if (!StringUtils.letters.includes(lower[0]) && lower[0] !== '_') {
+            return false;
+        }
+        return lower.split('').every((char) => StringUtils.word_valid_chars.includes(char));
+    }
+}
+StringUtils.quotes = ['"', "'", '`'];
+StringUtils.letters = 'abcdefghijklmnopqrstuvwxyz';
+StringUtils.digits = '0123456789';
+StringUtils.whitespace = ' \t\n\r';
+StringUtils.word_valid_chars = StringUtils.letters + StringUtils.digits + '_';
+exports.default = StringUtils;
+//# sourceMappingURL=string_utils.js.map

package/dist/utils/string_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"string_utils.js","sourceRoot":"","sources":["../../src/utils/string_utils.ts"],"names":[],"mappings":";;AAAA;;;;;GAKG;AACH,MAAM,WAAW;IAOb;;;;;OAKG;IACH,MAAM,CAAC,OAAO,CAAC,GAAW;QACtB,IAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClB,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAG,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtD,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,KAAK,GAAW,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAW,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzC,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACtD,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,SAAS,CAAC,GAAW;QACxB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACjG,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,mBAAmB,CAAC,GAAW,EAAE,SAAiB;QACrD,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,CAAC,EAAE,CAAC;YACR,CAAC;YACD,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,mBAAmB,CAAC,GAAW;QAClC,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBAClF,CAAC,EAAE,CAAC;YACR,CAAC;YACD,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,iBAAiB,CAAC,GAAW;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAG,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAG,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC7D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,CAAC;;AAtGe,kBAAM,GAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACnC,mBAAO,GAAG,4BAA4B,CAAC;AACvC,kBAAM,GAAG,YAAY,CAAC;AACtB,sBAAU,GAAG,SAAS,CAAC;AACvB,4BAAgB,GAAG,WAAW,CAAC,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC;AAsGtF,kBAAe,WAAW,CAAC"}