flowquery 1.0.0

package/src/tokenization/tokenizer.ts

@@ -0,0 +1,229 @@
+import Keyword from './keyword';
+import Token from './token';
+import StringWalker from './string_walker';
+import StringUtils from '../utils/string_utils';
+import Symbol from './symbol';
+import Operator from './operator';
+import TokenMapper from './token_mapper';
+
+/**
+ * Tokenizes FlowQuery input strings into a sequence of tokens.
+ * 
+ * The tokenizer performs lexical analysis, breaking down the input text into
+ * meaningful tokens such as keywords, identifiers, operators, strings, numbers,
+ * and symbols. It handles comments, whitespace, and f-strings.
+ * 
+ * @example
+ * ```typescript
+ * const tokenizer = new Tokenizer("WITH x = 1 RETURN x");
+ * const tokens = tokenizer.tokenize();
+ * ```
+ */
+class Tokenizer {
+    private walker: StringWalker;
+    private keywords: TokenMapper = new TokenMapper(Keyword);
+    private symbols: TokenMapper = new TokenMapper(Symbol);
+    private operators: TokenMapper = new TokenMapper(Operator);
+
+    /**
+     * Creates a new Tokenizer instance for the given input.
+     * 
+     * @param input - The FlowQuery input string to tokenize
+     */
+    constructor(input: string) {
+        this.walker = new StringWalker(input);
+    }
+
+    /**
+     * Tokenizes the input string into an array of tokens.
+     * 
+     * @returns An array of Token objects representing the tokenized input
+     * @throws {Error} If an unrecognized token is encountered
+     */
+    public tokenize(): Token[] {
+        const tokens: Token[] = [];
+        let last: Token | null = null;
+        while (!this.walker.isAtEnd) {
+            tokens.push(...this.f_string());
+            last = this.getLastNonWhitespaceOrNonCommentToken(tokens) || last;
+            const token = this.getNextToken(last);
+            if (token === null) {
+                throw new Error(`Unrecognized token at position ${this.walker.position}`);
+            }
+            token.position = this.walker.position;
+            tokens.push(token);
+        }
+        return tokens;
+    }
+
+    private getLastNonWhitespaceOrNonCommentToken(tokens: Token[]): Token | null {
+        if (tokens.length === 0) {
+            return null;
+        }
+        if(!tokens[tokens.length - 1].isWhitespaceOrComment()) {
+            return tokens[tokens.length - 1];
+        }
+        return null;
+    }
+
+    private getNextToken(last: Token | null = null): Token | null {
+        if (this.walker.isAtEnd) {
+            return Token.EOF;
+        }
+        return (
+            this.comment() ||
+            this.whitespace() ||
+            this.lookup(this.keywords) ||
+            this.lookup(this.operators, last, this.skipMinus) ||
+            this.identifier() ||
+            this.string() ||
+            this.number() ||
+            this.lookup(this.symbols)
+        );
+    }
+
+    public comment(): Token | null {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForSingleComment() || this.walker.checkForMultiLineComment()) {
+            const uncommented = StringUtils.uncomment(this.walker.getString(startPosition));
+            return Token.COMMENT(uncommented);
+        }
+        return null;
+    }
+
+    private identifier(): Token | null {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForUnderScore() || this.walker.checkForLetter()) {
+            while (!this.walker.isAtEnd && (this.walker.checkForLetter() || this.walker.checkForDigit() || this.walker.checkForUnderScore())) {
+                ;
+            }
+            return Token.IDENTIFIER(this.walker.getString(startPosition));
+        }
+        return null;
+    }
+
+    private string(): Token | null {
+        const startPosition = this.walker.position;
+        const quoteChar = this.walker.checkForQuote();
+        if (quoteChar === null) {
+            return null;
+        }
+        while (!this.walker.isAtEnd) {
+            if (this.walker.escaped(quoteChar)) {
+                this.walker.moveNext();
+                this.walker.moveNext();
+                continue;
+            }
+            if (this.walker.checkForString(quoteChar)) {
+                const value = this.walker.getString(startPosition);
+                if (quoteChar === Symbol.BACKTICK) {
+                    return Token.BACKTICK_STRING(value, quoteChar);
+                }
+                return Token.STRING(value, quoteChar);
+            }
+            this.walker.moveNext();
+        }
+        throw new Error(`Unterminated string at position ${startPosition}`);
+    }
+
+    private *f_string(): Iterable<Token> {
+        if(!this.walker.checkForFStringStart()) {
+            return;
+        }
+        this.walker.moveNext(); // skip the f
+        let position = this.walker.position;
+        const quoteChar = this.walker.checkForQuote();
+        if (quoteChar === null) {
+            return;
+        }
+        while (!this.walker.isAtEnd) {
+            if (this.walker.escaped(quoteChar) || this.walker.escapedBrace()) {
+                this.walker.moveNext();
+                this.walker.moveNext();
+                continue;
+            }
+            if(this.walker.openingBrace()) {
+                yield Token.F_STRING(this.walker.getString(position), quoteChar);
+                position = this.walker.position;
+                yield Token.OPENING_BRACE;
+                this.walker.moveNext(); // skip the opening brace
+                position = this.walker.position;
+                while(!this.walker.isAtEnd && !this.walker.closingBrace()) {
+                    const token = this.getNextToken();
+                    if(token !== null) {
+                        yield token;
+                    } else {
+                        break;
+                    }
+                    if(this.walker.closingBrace()) {
+                        yield Token.CLOSING_BRACE;
+                        this.walker.moveNext(); // skip the closing brace
+                        position = this.walker.position;
+                        break;
+                    }
+                }
+            }
+            if (this.walker.checkForString(quoteChar)) {
+                yield Token.F_STRING(this.walker.getString(position), quoteChar);
+                return;
+            };
+            this.walker.moveNext();
+        }
+    }
+
+    private whitespace(): Token | null {
+        let foundWhitespace = false;
+        while (!this.walker.isAtEnd && this.walker.checkForWhitespace()) {
+            this.walker.moveNext();
+            foundWhitespace = true;
+        }
+        return foundWhitespace ? Token.WHITESPACE : null;
+    }
+
+    private number(): Token | null {
+        const startPosition = this.walker.position;
+        if (this.walker.checkForString('-') || this.walker.checkForDigit()) {
+            while (!this.walker.isAtEnd && this.walker.checkForDigit()) {
+                ;
+            }
+            if (this.walker.checkForString(Symbol.DOT)) {
+                while (!this.walker.isAtEnd && this.walker.checkForDigit()) {
+                    ;
+                }
+            }
+            const _number = this.walker.getString(startPosition);
+            return Token.NUMBER(_number);
+        }
+        return null;
+    }
+
+    private lookup(mapper: TokenMapper, last: Token | null = null, skip?: (last: Token | null, current: Token) => boolean): Token | null {
+        const token = mapper.map(this.walker.getRemainingString());
+        if (token !== undefined && token.value !== null) {
+            if(token.can_be_identifier && this.walker.word_continuation(token.value)) {
+                return null;
+            }
+            if (skip && last && skip(last, token)) {
+                return null;
+            }
+            this.walker.moveBy(token.value.length);
+            if(mapper.last_found !== null) {
+                token.case_sensitive_value = mapper.last_found;
+            }
+            return token;
+        }
+        return null;
+    }
+
+    private skipMinus(last: Token | null, current: Token): boolean {
+        if (last === null) {
+            return false;
+        }
+        if((last.isKeyword() || last.isComma() || last.isColon()) && current.isNegation()) {
+            return true;
+        }
+        return false;
+    }
+}
+
+export default Tokenizer;

package/src/tokenization/trie.ts

@@ -0,0 +1,117 @@
+import Token from "./token";
+
+/**
+ * Represents a node in a Trie data structure.
+ * 
+ * Each node can have children nodes (one per character) and may contain a token
+ * if the path to this node represents a complete word.
+ */
+class Node {
+    private _children: Map<string, Node> = new Map();
+    private _token: Token | undefined = undefined;
+
+    public map(char: string): Node {
+        return this._children.get(char) || this._children.set(char, new Node()).get(char)!;
+    }
+
+    public retrieve(char: string): Node | undefined {
+        return this._children.get(char);
+    }
+
+    public set token(token: Token) {
+        this._token = token;
+    }
+
+    public get token(): Token | undefined {
+        return this._token;
+    }
+
+    public is_end_of_word(): boolean {
+        return this._token !== undefined;
+    }
+
+    public no_children(): boolean {
+        return this._children.size === 0;
+    }
+}
+
+/**
+ * Trie (prefix tree) data structure for efficient keyword and operator lookup.
+ * 
+ * Used during tokenization to quickly match input strings against known keywords
+ * and operators. Supports case-insensitive matching and tracks the longest match found.
+ * 
+ * @example
+ * ```typescript
+ * const trie = new Trie();
+ * trie.insert(Token.WITH);
+ * const found = trie.find("WITH");
+ * ```
+ */
+class Trie {
+    private _root: Node = new Node();
+    private _max_length: number = 0;
+    private _last_found: string | null = null;
+
+    /**
+     * Inserts a token into the trie.
+     * 
+     * @param token - The token to insert
+     * @throws {Error} If the token value is null or empty
+     */
+    public insert(token: Token): void {
+        if(token.value === null || token.value.length === 0) {
+            throw new Error("Token value cannot be null or empty");
+        }
+        let currentNode = this._root;
+        for (const char of token.value) {
+            currentNode = currentNode.map(char.toLowerCase());
+        }
+        if (token.value.length > this._max_length) {
+            this._max_length = token.value.length;
+        }
+        currentNode.token = token;
+    }
+
+    /**
+     * Finds a token by searching for the longest matching prefix in the trie.
+     * 
+     * @param value - The string value to search for
+     * @returns The token if found, undefined otherwise
+     */
+    public find(value: string): Token | undefined {
+        if(value.length === 0) {
+            return undefined;
+        }
+        let index = 0;
+        let current: Node | undefined = undefined;
+        let found: Token | undefined = undefined;
+        this._last_found = null;
+        while((current = (current || this._root).retrieve(value[index].toLowerCase())) !== undefined) {
+            if(current.is_end_of_word()) {
+                found = current.token;
+                this._last_found = value.substring(0, index + 1);
+            }
+            index++;
+            if(index === value.length || index > this._max_length) {
+                break;
+            }
+        }
+        if(current?.is_end_of_word()) {
+            found = current.token;
+            this._last_found = value.substring(0, index);
+        }
+        return found;
+    }
+
+    /**
+     * Gets the last matched string from the most recent find operation.
+     * 
+     * @returns The last found string, or null if no match was found
+     */
+    public get last_found(): string | null {
+        return this._last_found;
+    }
+}
+
+export default Trie;

package/src/utils/object_utils.ts

@@ -0,0 +1,17 @@
+/**
+ * Utility class for object-related operations.
+ */
+class ObjectUtils {
+    /**
+     * Checks if an object is an instance of any of the provided classes.
+     * 
+     * @param obj - The object to check
+     * @param classes - Array of class constructors to test against
+     * @returns True if the object is an instance of any class, false otherwise
+     */
+    static isInstanceOfAny(obj: any, classes: any[]): boolean {
+        return classes.some(cls => obj instanceof cls);
+    }
+}
+
+export default ObjectUtils;

package/src/utils/string_utils.ts

@@ -0,0 +1,114 @@
+/**
+ * Utility class for string manipulation and validation.
+ * 
+ * Provides methods for handling quoted strings, comments, escape sequences,
+ * and identifier validation.
+ */
+class StringUtils {
+    static readonly quotes: string[] = ['"', "'", '`'];
+    static readonly letters = 'abcdefghijklmnopqrstuvwxyz';
+    static readonly digits = '0123456789';
+    static readonly whitespace = ' \t\n\r';
+    static readonly word_valid_chars = StringUtils.letters + StringUtils.digits + '_';
+
+    /**
+     * Removes surrounding quotes from a string.
+     * 
+     * @param str - The string to unquote
+     * @returns The unquoted string
+     */
+    static unquote(str: string): string {
+        if(str.length === 0) {
+            return str;
+        }
+        if(str.length === 1 && StringUtils.quotes.includes(str)) {
+            return '';
+        }
+        const first: string = str[0];
+        const last: string = str[str.length - 1];
+        if (StringUtils.quotes.includes(first) && first === last) {
+            return str.substring(1, str.length - 1);
+        }
+        if (StringUtils.quotes.includes(last) && first !== last) {
+            return str.substring(0, str.length - 1);
+        }
+        if (StringUtils.quotes.includes(first) && first !== last) {
+            return str.substring(1);
+        }
+        return str;
+    }
+
+    /**
+     * Removes comment markers from a string.
+     * 
+     * @param str - The comment string
+     * @returns The string without comment markers
+     */
+    static uncomment(str: string): string {
+        if (str.length < 2) {
+            return str;
+        }
+        if (str[0] === '/' && str[1] === '/') {
+            return str.substring(2);
+        }
+        if (str[0] === '/' && str[1] === '*' && str[str.length - 2] === '*' && str[str.length - 1] === '/') {
+            return str.substring(2, str.length - 2);
+        }
+        return str;
+    }
+
+    /**
+     * Removes escape sequences before quotes in a string.
+     * 
+     * @param str - The string to process
+     * @param quoteChar - The quote character that was escaped
+     * @returns The string with escape sequences removed
+     */
+    static removeEscapedQuotes(str: string, quoteChar: string): string {
+        let unescaped = '';
+        for (let i = 0; i < str.length; i++) {
+            if (str[i] === '\\' && str[i + 1] === quoteChar) {
+                i++;
+            }
+            unescaped += str[i];
+        }
+        return unescaped;
+    }
+
+    /**
+     * Removes escaped braces ({{ and }}) from f-strings.
+     * 
+     * @param str - The string to process
+     * @returns The string with escaped braces resolved
+     */
+    static removeEscapedBraces(str: string): string {
+        let unescaped = '';
+        for (let i = 0; i < str.length; i++) {
+            if((str[i] === '{' && str[i + 1] === '{') || (str[i] === '}' && str[i + 1] === '}')) {
+                i++;
+            }
+            unescaped += str[i];
+        }
+        return unescaped;
+    }
+
+    /**
+     * Checks if a string is a valid identifier.
+     * 
+     * @param str - The string to validate
+     * @returns True if the string can be used as an identifier, false otherwise
+     */
+    static can_be_identifier(str: string): boolean {
+        const lower = str.toLowerCase();
+        if(lower.length === 0) {
+            return false;
+        }
+        if(!StringUtils.letters.includes(lower[0]) && lower[0] !== '_') {
+            return false;
+        }
+        return lower.split('').every((char) => StringUtils.word_valid_chars.includes(char));
+    }
+
+}
+
+export default StringUtils;