flowquery 1.0.46 → 1.0.47

package/src/tokenization/token_mapper.ts

@@ -1,53 +0,0 @@
-import Token from "./token";
-import Trie from "./trie";
-
-/**
- * Maps string values to tokens using a Trie for efficient lookup.
- * 
- * Takes an enum of keywords, operators, or symbols and builds a trie
- * for fast token matching during tokenization.
- * 
- * @example
- * ```typescript
- * const mapper = new TokenMapper(Keyword);
- * const token = mapper.map("WITH");
- * ```
- */
-class TokenMapper {
-    private _trie: Trie = new Trie();
-    
-    /**
-     * Creates a TokenMapper from an enum of token values.
-     * 
-     * @param _enum - An enum object containing token values
-     */
-    constructor(private _enum: { [key: string]: any }) {
-        for(const [key, value] of Object.entries(_enum)) {
-            const token: Token | undefined = Token.method(key);
-            if(token !== undefined && token.value !== null) {
-                this._trie.insert(token);
-            }
-        }
-    }
-    
-    /**
-     * Maps a string value to its corresponding token.
-     * 
-     * @param value - The string value to map
-     * @returns The matched token, or undefined if no match found
-     */
-    public map(value: string): Token | undefined {
-        return this._trie.find(value);
-    }
-    
-    /**
-     * Gets the last matched string from the most recent map operation.
-     * 
-     * @returns The last found string, or null if no match
-     */
-    public get last_found(): string | null {
-        return this._trie.last_found;
-    }
-}
-
-export default TokenMapper;

package/src/tokenization/token_type.ts

@@ -1,16 +0,0 @@
-enum TokenType {
-    KEYWORD = "KEYWORD",
-    BOOLEAN = "BOOLEAN",
-    OPERATOR = "OPERATOR",
-    UNARY_OPERATOR = "UNARY_OPERATOR",
-    IDENTIFIER = "IDENTIFIER",
-    STRING = "STRING",
-    F_STRING = "F-STRING",
-    BACKTICK_STRING = "BACKTICK_STRING",
-    NUMBER = "NUMBER",
-    SYMBOL = "SYMBOL",
-    WHITESPACE = "WHITESPACE",
-    COMMENT = "COMMENT",
-    EOF = "EOF",
-}
-export default TokenType;

package/src/tokenization/tokenizer.ts

@@ -1,250 +0,0 @@
-import StringUtils from "../utils/string_utils";
-import Keyword from "./keyword";
-import Operator from "./operator";
-import StringWalker from "./string_walker";
-import Symbol from "./symbol";
-import Token from "./token";
-import TokenMapper from "./token_mapper";
-
-/**
- * Tokenizes FlowQuery input strings into a sequence of tokens.
- *
- * The tokenizer performs lexical analysis, breaking down the input text into
- * meaningful tokens such as keywords, identifiers, operators, strings, numbers,
- * and symbols. It handles comments, whitespace, and f-strings.
- *
- * @example
- * ```typescript
- * const tokenizer = new Tokenizer("WITH x = 1 RETURN x");
- * const tokens = tokenizer.tokenize();
- * ```
- */
-class Tokenizer {
-    private walker: StringWalker;
-    private keywords: TokenMapper = new TokenMapper(Keyword);
-    private symbols: TokenMapper = new TokenMapper(Symbol);
-    private operators: TokenMapper = new TokenMapper(Operator);
-
-    /**
-     * Creates a new Tokenizer instance for the given input.
-     *
-     * @param input - The FlowQuery input string to tokenize
-     */
-    constructor(input: string) {
-        this.walker = new StringWalker(input);
-    }
-
-    /**
-     * Tokenizes the input string into an array of tokens.
-     *
-     * @returns An array of Token objects representing the tokenized input
-     * @throws {Error} If an unrecognized token is encountered
-     */
-    public tokenize(): Token[] {
-        const tokens: Token[] = [];
-        let last: Token | null = null;
-        while (!this.walker.isAtEnd) {
-            tokens.push(...this.f_string());
-            last = this.getLastNonWhitespaceOrNonCommentToken(tokens) || last;
-            const token = this.getNextToken(last);
-            if (token === null) {
-                throw new Error(`Unrecognized token at position ${this.walker.position}`);
-            }
-            token.position = this.walker.position;
-            tokens.push(token);
-        }
-        return tokens;
-    }
-
-    private getLastNonWhitespaceOrNonCommentToken(tokens: Token[]): Token | null {
-        if (tokens.length === 0) {
-            return null;
-        }
-        if (!tokens[tokens.length - 1].isWhitespaceOrComment()) {
-            return tokens[tokens.length - 1];
-        }
-        return null;
-    }
-
-    private getNextToken(last: Token | null = null): Token | null {
-        if (this.walker.isAtEnd) {
-            return Token.EOF;
-        }
-        return (
-            this.comment() ||
-            this.whitespace() ||
-            this.lookup(this.keywords) ||
-            this.lookup(this.operators, last, this.skipMinus) ||
-            this.boolean() ||
-            this.identifier() ||
-            this.string() ||
-            this.number() ||
-            this.lookup(this.symbols)
-        );
-    }
-
-    public comment(): Token | null {
-        const startPosition = this.walker.position;
-        if (this.walker.checkForSingleComment() || this.walker.checkForMultiLineComment()) {
-            const uncommented = StringUtils.uncomment(this.walker.getString(startPosition));
-            return Token.COMMENT(uncommented);
-        }
-        return null;
-    }
-
-    private boolean(): Token | null {
-        const startPosition = this.walker.position;
-        if (this.walker.checkForString("TRUE")) {
-            return Token.BOOLEAN(this.walker.getString(startPosition).toUpperCase());
-        }
-        if (this.walker.checkForString("FALSE")) {
-            return Token.BOOLEAN(this.walker.getString(startPosition).toUpperCase());
-        }
-        return null;
-    }
-
-    private identifier(): Token | null {
-        const startPosition = this.walker.position;
-        if (this.walker.checkForUnderScore() || this.walker.checkForLetter()) {
-            while (
-                !this.walker.isAtEnd &&
-                (this.walker.checkForLetter() ||
-                    this.walker.checkForDigit() ||
-                    this.walker.checkForUnderScore())
-            ) {}
-            return Token.IDENTIFIER(this.walker.getString(startPosition));
-        }
-        return null;
-    }
-
-    private string(): Token | null {
-        const startPosition = this.walker.position;
-        const quoteChar = this.walker.checkForQuote();
-        if (quoteChar === null) {
-            return null;
-        }
-        while (!this.walker.isAtEnd) {
-            if (this.walker.escaped(quoteChar)) {
-                this.walker.moveNext();
-                this.walker.moveNext();
-                continue;
-            }
-            if (this.walker.checkForString(quoteChar)) {
-                const value = this.walker.getString(startPosition);
-                if (quoteChar === Symbol.BACKTICK) {
-                    return Token.BACKTICK_STRING(value, quoteChar);
-                }
-                return Token.STRING(value, quoteChar);
-            }
-            this.walker.moveNext();
-        }
-        throw new Error(`Unterminated string at position ${startPosition}`);
-    }
-
-    private *f_string(): Iterable<Token> {
-        if (!this.walker.checkForFStringStart()) {
-            return;
-        }
-        this.walker.moveNext(); // skip the f
-        let position = this.walker.position;
-        const quoteChar = this.walker.checkForQuote();
-        if (quoteChar === null) {
-            return;
-        }
-        while (!this.walker.isAtEnd) {
-            if (this.walker.escaped(quoteChar) || this.walker.escapedBrace()) {
-                this.walker.moveNext();
-                this.walker.moveNext();
-                continue;
-            }
-            if (this.walker.openingBrace()) {
-                yield Token.F_STRING(this.walker.getString(position), quoteChar);
-                position = this.walker.position;
-                yield Token.OPENING_BRACE;
-                this.walker.moveNext(); // skip the opening brace
-                position = this.walker.position;
-                while (!this.walker.isAtEnd && !this.walker.closingBrace()) {
-                    const token = this.getNextToken();
-                    if (token !== null) {
-                        yield token;
-                    } else {
-                        break;
-                    }
-                    if (this.walker.closingBrace()) {
-                        yield Token.CLOSING_BRACE;
-                        this.walker.moveNext(); // skip the closing brace
-                        position = this.walker.position;
-                        break;
-                    }
-                }
-            }
-            if (this.walker.checkForString(quoteChar)) {
-                yield Token.F_STRING(this.walker.getString(position), quoteChar);
-                return;
-            }
-            this.walker.moveNext();
-        }
-    }
-
-    private whitespace(): Token | null {
-        let foundWhitespace = false;
-        while (!this.walker.isAtEnd && this.walker.checkForWhitespace()) {
-            this.walker.moveNext();
-            foundWhitespace = true;
-        }
-        return foundWhitespace ? Token.WHITESPACE : null;
-    }
-
-    private number(): Token | null {
-        const startPosition = this.walker.position;
-        if (this.walker.checkForString("-") || this.walker.checkForDigit()) {
-            while (!this.walker.isAtEnd && this.walker.checkForDigit()) {}
-            if (this.walker.checkForString(Symbol.DOT)) {
-                let decimalDigits: number = 0;
-                while (!this.walker.isAtEnd && this.walker.checkForDigit()) {
-                    decimalDigits++;
-                }
-                if (decimalDigits === 0) {
-                    this.walker.movePrevious();
-                }
-            }
-            const _number = this.walker.getString(startPosition);
-            return Token.NUMBER(_number);
-        }
-        return null;
-    }
-
-    private lookup(
-        mapper: TokenMapper,
-        last: Token | null = null,
-        skip?: (last: Token | null, current: Token) => boolean
-    ): Token | null {
-        const token = mapper.map(this.walker.getRemainingString());
-        if (token !== undefined && token.value !== null) {
-            if (token.can_be_identifier && this.walker.word_continuation(token.value)) {
-                return null;
-            }
-            if (skip && last && skip(last, token)) {
-                return null;
-            }
-            this.walker.moveBy(token.value.length);
-            if (mapper.last_found !== null) {
-                token.case_sensitive_value = mapper.last_found;
-            }
-            return token;
-        }
-        return null;
-    }
-
-    private skipMinus(last: Token | null, current: Token): boolean {
-        if (last === null) {
-            return false;
-        }
-        if ((last.isKeyword() || last.isComma() || last.isColon()) && current.isNegation()) {
-            return true;
-        }
-        return false;
-    }
-}
-
-export default Tokenizer;

package/src/tokenization/trie.ts

@@ -1,117 +0,0 @@
-import Token from "./token";
-
-/**
- * Represents a node in a Trie data structure.
- * 
- * Each node can have children nodes (one per character) and may contain a token
- * if the path to this node represents a complete word.
- */
-class Node {
-    private _children: Map<string, Node> = new Map();
-    private _token: Token | undefined = undefined;
-
-    public map(char: string): Node {
-        return this._children.get(char) || this._children.set(char, new Node()).get(char)!;
-    }
-
-    public retrieve(char: string): Node | undefined {
-        return this._children.get(char);
-    }
-
-    public set token(token: Token) {
-        this._token = token;
-    }
-
-    public get token(): Token | undefined {
-        return this._token;
-    }
-
-    public is_end_of_word(): boolean {
-        return this._token !== undefined;
-    }
-
-    public no_children(): boolean {
-        return this._children.size === 0;
-    }
-}
-
-/**
- * Trie (prefix tree) data structure for efficient keyword and operator lookup.
- * 
- * Used during tokenization to quickly match input strings against known keywords
- * and operators. Supports case-insensitive matching and tracks the longest match found.
- * 
- * @example
- * ```typescript
- * const trie = new Trie();
- * trie.insert(Token.WITH);
- * const found = trie.find("WITH");
- * ```
- */
-class Trie {
-    private _root: Node = new Node();
-    private _max_length: number = 0;
-    private _last_found: string | null = null;
-
-    /**
-     * Inserts a token into the trie.
-     * 
-     * @param token - The token to insert
-     * @throws {Error} If the token value is null or empty
-     */
-    public insert(token: Token): void {
-        if(token.value === null || token.value.length === 0) {
-            throw new Error("Token value cannot be null or empty");
-        }
-        let currentNode = this._root;
-        for (const char of token.value) {
-            currentNode = currentNode.map(char.toLowerCase());
-        }
-        if (token.value.length > this._max_length) {
-            this._max_length = token.value.length;
-        }
-        currentNode.token = token;
-    }
-
-    /**
-     * Finds a token by searching for the longest matching prefix in the trie.
-     * 
-     * @param value - The string value to search for
-     * @returns The token if found, undefined otherwise
-     */
-    public find(value: string): Token | undefined {
-        if(value.length === 0) {
-            return undefined;
-        }
-        let index = 0;
-        let current: Node | undefined = undefined;
-        let found: Token | undefined = undefined;
-        this._last_found = null;
-        while((current = (current || this._root).retrieve(value[index].toLowerCase())) !== undefined) {
-            if(current.is_end_of_word()) {
-                found = current.token;
-                this._last_found = value.substring(0, index + 1);
-            }
-            index++;
-            if(index === value.length || index > this._max_length) {
-                break;
-            }
-        }
-        if(current?.is_end_of_word()) {
-            found = current.token;
-            this._last_found = value.substring(0, index);
-        }
-        return found;
-    }
-
-    /**
-     * Gets the last matched string from the most recent find operation.
-     * 
-     * @returns The last found string, or null if no match was found
-     */
-    public get last_found(): string | null {
-        return this._last_found;
-    }
-}
-
-export default Trie;

package/src/utils/object_utils.ts

@@ -1,17 +0,0 @@
-/**
- * Utility class for object-related operations.
- */
-class ObjectUtils {
-    /**
-     * Checks if an object is an instance of any of the provided classes.
-     * 
-     * @param obj - The object to check
-     * @param classes - Array of class constructors to test against
-     * @returns True if the object is an instance of any class, false otherwise
-     */
-    static isInstanceOfAny(obj: any, classes: any[]): boolean {
-        return classes.some(cls => obj instanceof cls);
-    }
-}
-
-export default ObjectUtils;

package/src/utils/string_utils.ts

@@ -1,114 +0,0 @@
-/**
- * Utility class for string manipulation and validation.
- * 
- * Provides methods for handling quoted strings, comments, escape sequences,
- * and identifier validation.
- */
-class StringUtils {
-    static readonly quotes: string[] = ['"', "'", '`'];
-    static readonly letters = 'abcdefghijklmnopqrstuvwxyz';
-    static readonly digits = '0123456789';
-    static readonly whitespace = ' \t\n\r';
-    static readonly word_valid_chars = StringUtils.letters + StringUtils.digits + '_';
-
-    /**
-     * Removes surrounding quotes from a string.
-     * 
-     * @param str - The string to unquote
-     * @returns The unquoted string
-     */
-    static unquote(str: string): string {
-        if(str.length === 0) {
-            return str;
-        }
-        if(str.length === 1 && StringUtils.quotes.includes(str)) {
-            return '';
-        }
-        const first: string = str[0];
-        const last: string = str[str.length - 1];
-        if (StringUtils.quotes.includes(first) && first === last) {
-            return str.substring(1, str.length - 1);
-        }
-        if (StringUtils.quotes.includes(last) && first !== last) {
-            return str.substring(0, str.length - 1);
-        }
-        if (StringUtils.quotes.includes(first) && first !== last) {
-            return str.substring(1);
-        }
-        return str;
-    }
-
-    /**
-     * Removes comment markers from a string.
-     * 
-     * @param str - The comment string
-     * @returns The string without comment markers
-     */
-    static uncomment(str: string): string {
-        if (str.length < 2) {
-            return str;
-        }
-        if (str[0] === '/' && str[1] === '/') {
-            return str.substring(2);
-        }
-        if (str[0] === '/' && str[1] === '*' && str[str.length - 2] === '*' && str[str.length - 1] === '/') {
-            return str.substring(2, str.length - 2);
-        }
-        return str;
-    }
-
-    /**
-     * Removes escape sequences before quotes in a string.
-     * 
-     * @param str - The string to process
-     * @param quoteChar - The quote character that was escaped
-     * @returns The string with escape sequences removed
-     */
-    static removeEscapedQuotes(str: string, quoteChar: string): string {
-        let unescaped = '';
-        for (let i = 0; i < str.length; i++) {
-            if (str[i] === '\\' && str[i + 1] === quoteChar) {
-                i++;
-            }
-            unescaped += str[i];
-        }
-        return unescaped;
-    }
-
-    /**
-     * Removes escaped braces ({{ and }}) from f-strings.
-     * 
-     * @param str - The string to process
-     * @returns The string with escaped braces resolved
-     */
-    static removeEscapedBraces(str: string): string {
-        let unescaped = '';
-        for (let i = 0; i < str.length; i++) {
-            if((str[i] === '{' && str[i + 1] === '{') || (str[i] === '}' && str[i + 1] === '}')) {
-                i++;
-            }
-            unescaped += str[i];
-        }
-        return unescaped;
-    }
-
-    /**
-     * Checks if a string is a valid identifier.
-     * 
-     * @param str - The string to validate
-     * @returns True if the string can be used as an identifier, false otherwise
-     */
-    static can_be_identifier(str: string): boolean {
-        const lower = str.toLowerCase();
-        if(lower.length === 0) {
-            return false;
-        }
-        if(!StringUtils.letters.includes(lower[0]) && lower[0] !== '_') {
-            return false;
-        }
-        return lower.split('').every((char) => StringUtils.word_valid_chars.includes(char));
-    }
-
-}
-
-export default StringUtils;