flowquery 1.0.0

package/src/parsing/token_to_node.ts

@@ -0,0 +1,91 @@
+import { Add, Subtract, Multiply, Divide, Modulo, Power, Equals, LessThan, GreaterThan, GreaterThanOrEqual, LessThanOrEqual, And, Or, Not, Is } from './expressions/operator';
+import String from './expressions/string';
+import Identifier from './expressions/identifier';
+import Number from './expressions/number';
+import Token from '../tokenization/token';
+import ASTNode from './ast_node';
+import JSON from './components/json';
+import CSV from './components/csv';
+import Text from './components/text';
+import When from './logic/when';
+import Then from './logic/then';
+import Else from './logic/else';
+import End from './logic/end';
+import Null from './components/null';
+class TokenToNode {
+    public static convert(token: Token): ASTNode {
+        if(token.isNumber()) {
+            if(token.value === null) {
+                throw new Error('Number token has no value');
+            }
+            return new Number(token.value);
+        } else if(token.isString()) {
+            if(token.value === null) {
+                throw new Error('String token has no value');
+            }
+            return new String(token.value);
+        } else if(token.isIdentifier()) {
+            if(token.value === null) {
+                throw new Error('Identifier token has no value');
+            }
+            return new Identifier(token.value);
+        } else if(token.isOperator()) {
+            if(token.isAdd()) {
+                return new Add();
+            } else if(token.isSubtract()) {
+                return new Subtract();
+            } else if(token.isMultiply()) {
+                return new Multiply();
+            } else if(token.isDivide()) {
+                return new Divide();
+            } else if(token.isModulo()) {
+                return new Modulo();
+            } else if(token.isExponent()) {
+                return new Power();
+            } else if(token.isEquals()) {
+                return new Equals();
+            } else if(token.isLessThan()) {
+                return new LessThan();
+            } else if(token.isGreaterThan()) {
+                return new GreaterThan();
+            } else if(token.isGreaterThanOrEqual()) {
+                return new GreaterThanOrEqual();
+            } else if(token.isLessThanOrEqual()) {
+                return new LessThanOrEqual();
+            } else if(token.isAnd()) {
+                return new And();
+            } else if(token.isOr()) {
+                return new Or();
+            } else if(token.isIs()) {
+                return new Is();
+            }
+        } else if(token.isUnaryOperator()) {
+            if(token.isNot()) {
+                return new Not();
+            }
+        } else if (token.isKeyword()) {
+            if(token.isJSON()) {
+                return new JSON();
+            } else if(token.isCSV()) {
+                return new CSV();
+            } else if(token.isText()) {
+                return new Text();
+            } else if(token.isWhen()) {
+                return new When();
+            } else if(token.isThen()) {
+                return new Then();
+            } else if(token.isElse()) {
+                return new Else();
+            } else if(token.isEnd()) {
+                return new End();
+            } else if(token.isNull()) {
+                return new Null();
+            }
+        } else {
+            throw new Error('Unknown token');
+        }
+        return new ASTNode();
+    }
+}
+
+export default TokenToNode;

package/src/tokenization/keyword.ts

@@ -0,0 +1,43 @@
+enum Keyword {
+    RETURN = 'RETURN',
+    MATCH = 'MATCH',
+    WHERE = 'WHERE',
+    CREATE = 'CREATE',
+    MERGE = 'MERGE',
+    DELETE = 'DELETE',
+    DETACH = 'DETACH',
+    SET = 'SET',
+    REMOVE = 'REMOVE',
+    FOREACH = 'FOREACH',
+    WITH = 'WITH',
+    CALL = 'CALL',
+    YIELD = 'YIELD',
+    LOAD = 'LOAD',
+    HEADERS = 'HEADERS',
+    POST = 'POST',
+    FROM = 'FROM',
+    CSV = 'CSV',
+    JSON = 'JSON',
+    TEXT = 'TEXT',
+    AS = 'AS',
+    UNWIND = 'UNWIND',
+    SUM = 'SUM',
+    COLLECT = 'COLLECT',
+    DISTINCT = 'DISTINCT',
+    ORDER = 'ORDER',
+    BY = 'BY',
+    ASC = 'ASC',
+    DESC = 'DESC',
+    SKIP = 'SKIP',
+    LIMIT = 'LIMIT',
+    EOF = 'EOF',
+    CASE = 'CASE',
+    WHEN = 'WHEN',
+    THEN = 'THEN',
+    ELSE = 'ELSE',
+    END = 'END',
+    NULL = 'NULL',
+    IN = 'IN',
+}
+
+export default Keyword;

package/src/tokenization/operator.ts

@@ -0,0 +1,25 @@
+enum Operator {
+    // Arithmetic
+    ADD = '+',
+    SUBTRACT = '-',
+    MULTIPLY = '*',
+    DIVIDE = '/',
+    MODULO = '%',
+    EXPONENT = '^',
+    // Comparison
+    EQUALS = '=',
+    NOT_EQUALS = '<>',
+    LESS_THAN = '<',
+    LESS_THAN_OR_EQUAL = '<=',
+    GREATER_THAN = '>',
+    GREATER_THAN_OR_EQUAL = '>=',
+    IS = 'IS',
+    // Logical
+    AND = 'AND',
+    OR = 'OR',
+    NOT = 'NOT',
+    IN = 'IN',
+    PIPE = '|',
+}
+
+export default Operator;

package/src/tokenization/string_walker.ts

@@ -0,0 +1,194 @@
+import StringUtils from "../utils/string_utils";
+
+/**
+ * Utility class for walking through a string character by character during tokenization.
+ * 
+ * Provides methods to check for specific character patterns, move through the string,
+ * and extract substrings. Used by the Tokenizer to process input text.
+ * 
+ * @example
+ * ```typescript
+ * const walker = new StringWalker("WITH x as variable");
+ * while (!walker.isAtEnd) {
+ *   // Process characters
+ * }
+ * ```
+ */
+class StringWalker {
+    private _position: number;
+    private readonly text: string;
+
+    /**
+     * Creates a new StringWalker for the given text.
+     * 
+     * @param text - The input text to walk through
+     */
+    constructor(text: string) {
+        this.text = text;
+        this._position = 0;
+    }
+
+    public get position(): number {
+        return this._position;
+    }
+
+    public get currentChar(): string {
+        return this.text[this.position];
+    }
+
+    public get nextChar(): string {
+        return this.text[this.position + 1];
+    }
+
+    public get previousChar(): string {
+        return this.text[this.position - 1];
+    }
+
+    public get isAtEnd(): boolean {
+        return this.position >= this.text.length;
+    }
+
+    public getString(startPosition: number): string {
+        return this.text.substring(startPosition, this.position);
+    }
+
+    public getRemainingString(): string {
+        return this.text.substring(this.position);
+    }
+
+    public checkForSingleComment(): boolean {
+        if (this.singleLineCommentStart()) {
+            while (!this.isAtEnd && !this.newLine()) {
+                this._position++;
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public checkForMultiLineComment(): boolean {
+        if (this.multiLineCommentStart()) {
+            while (!this.isAtEnd) {
+                if (this.multiLineCommentEnd()) {
+                    this._position += 2;
+                    return true;
+                }
+                this._position++;
+            }
+            throw new Error(`Unterminated multi-line comment at position ${this.position}`);
+        }
+        return false;
+    }
+
+    public singleLineCommentStart(): boolean {
+        return this.currentChar === '/' && this.nextChar === '/';
+    }
+
+    public multiLineCommentStart(): boolean {
+        return this.currentChar === '/' && this.nextChar === '*';
+    }
+
+    public multiLineCommentEnd(): boolean {
+        return this.currentChar === '*' && this.nextChar === '/';
+    }
+
+    public newLine(): boolean {
+        if (this.currentChar === '\n') {
+            return true;
+        }
+        return false;
+    }
+
+    public escaped(char: string): boolean {
+        return this.currentChar === '\\' && this.nextChar === char;
+    }
+
+    public escapedBrace(): boolean {
+        return (this.currentChar === '{' && this.nextChar === '{') || (this.currentChar === '}' && this.nextChar === '}');
+    }
+
+    public openingBrace(): boolean {
+        return this.currentChar === '{';
+    }
+
+    public closingBrace(): boolean {
+        return this.currentChar === '}';
+    }
+
+    public checkForUnderScore(): boolean {
+        const foundUnderScore = this.currentChar === '_';
+        if (foundUnderScore) {
+            this._position++;
+        }
+        return foundUnderScore;
+    }
+
+    public checkForLetter(): boolean {
+        const foundLetter = StringUtils.letters.includes(this.currentChar.toLowerCase());
+        if (foundLetter) {
+            this._position++;
+        }
+        return foundLetter;
+    }
+
+    public checkForDigit(): boolean {
+        const foundDigit = StringUtils.digits.includes(this.currentChar);
+        if (foundDigit) {
+            this._position++;
+        }
+        return foundDigit;
+    }
+
+    public checkForQuote(): string | null {
+        const quoteChar = this.currentChar;
+        if (quoteChar === '"' || quoteChar === "'" || quoteChar === '`') {
+            this._position++;
+            return quoteChar;
+        }
+        return null;
+    }
+
+    public checkForString(value: string): boolean {
+        const _string = this.text.substring(this.position, this.position + value.length);
+
+        const foundString = _string.toLowerCase() === value.toLowerCase();
+        if (foundString) {
+            this._position += value.length;
+        }
+        return foundString;
+    }
+
+    public checkForWhitespace(): boolean {
+        return StringUtils.whitespace.includes(this.currentChar);
+    }
+
+    public checkForFStringStart(): boolean {
+        return this.currentChar.toLowerCase() === 'f' && ['\'', '"', '`'].includes(this.nextChar);
+    }
+
+    public moveNext(): void {
+        this._position++;
+    }
+
+    public moveBy(steps: number): void {
+        this._position += steps;
+    }
+
+    public movePrevious(): void {
+        this._position--;
+    }
+
+    public is_word(word: string | null): boolean {
+        if (word === null) {
+            return false;
+        }
+        return this.text.substring(this.position, this.position + word.length) === word;
+    }
+
+    public word_continuation(word: string): boolean {
+        const next = this.text[this.position + word.length];
+        return StringUtils.word_valid_chars.includes(next);
+    }
+}
+
+export default StringWalker;

package/src/tokenization/symbol.ts

@@ -0,0 +1,15 @@
+enum Symbol {
+    LEFT_PARENTHESIS = '(',
+    RIGHT_PARENTHESIS = ')',
+    COMMA = ',',
+    DOT = '.',
+    COLON = ':',
+    WHITESPACE = '',
+    OPENING_BRACE = '{',
+    CLOSING_BRACE = '}',
+    OPENING_BRACKET = '[',
+    CLOSING_BRACKET = ']',
+    BACKTICK = '`',
+}
+
+export default Symbol;