flowquery 1.0.16 → 1.0.17

package/flowquery-py/src/parsing/token_to_node.py

@@ -0,0 +1,109 @@
+"""Converts tokens to AST nodes."""
+
+from ..tokenization.token import Token
+from .ast_node import ASTNode
+from .components.csv import CSV
+from .components.json import JSON
+from .components.null import Null
+from .components.text import Text
+from .expressions.boolean import Boolean
+from .expressions.identifier import Identifier
+from .expressions.number import Number
+from .expressions.string import String
+from .expressions.operator import (
+    Add,
+    And,
+    Divide,
+    Equals,
+    GreaterThan,
+    GreaterThanOrEqual,
+    Is,
+    LessThan,
+    LessThanOrEqual,
+    Modulo,
+    Multiply,
+    Not,
+    NotEquals,
+    Or,
+    Power,
+    Subtract,
+)
+from .logic.else_ import Else
+from .logic.end import End
+from .logic.then import Then
+from .logic.when import When
+
+
+class TokenToNode:
+    """Converts tokens to their corresponding AST nodes."""
+
+    @staticmethod
+    def convert(token: Token) -> ASTNode:
+        if token.is_number():
+            if token.value is None:
+                raise ValueError("Number token has no value")
+            return Number(token.value)
+        elif token.is_string():
+            if token.value is None:
+                raise ValueError("String token has no value")
+            return String(token.value)
+        elif token.is_identifier():
+            if token.value is None:
+                raise ValueError("Identifier token has no value")
+            return Identifier(token.value)
+        elif token.is_operator():
+            if token.is_add():
+                return Add()
+            elif token.is_subtract():
+                return Subtract()
+            elif token.is_multiply():
+                return Multiply()
+            elif token.is_divide():
+                return Divide()
+            elif token.is_modulo():
+                return Modulo()
+            elif token.is_exponent():
+                return Power()
+            elif token.is_equals():
+                return Equals()
+            elif token.is_not_equals():
+                return NotEquals()
+            elif token.is_less_than():
+                return LessThan()
+            elif token.is_greater_than():
+                return GreaterThan()
+            elif token.is_greater_than_or_equal():
+                return GreaterThanOrEqual()
+            elif token.is_less_than_or_equal():
+                return LessThanOrEqual()
+            elif token.is_and():
+                return And()
+            elif token.is_or():
+                return Or()
+            elif token.is_is():
+                return Is()
+        elif token.is_unary_operator():
+            if token.is_not():
+                return Not()
+        elif token.is_keyword():
+            if token.is_json():
+                return JSON()
+            elif token.is_csv():
+                return CSV()
+            elif token.is_text():
+                return Text()
+            elif token.is_when():
+                return When()
+            elif token.is_then():
+                return Then()
+            elif token.is_else():
+                return Else()
+            elif token.is_end():
+                return End()
+            elif token.is_null():
+                return Null()
+        elif token.is_boolean():
+            return Boolean(token.value)
+        else:
+            raise ValueError("Unknown token")
+        return ASTNode()

package/flowquery-py/src/tokenization/__init__.py

@@ -0,0 +1,23 @@
+"""Tokenization module for FlowQuery."""
+
+from .tokenizer import Tokenizer
+from .token import Token
+from .token_type import TokenType
+from .keyword import Keyword
+from .operator import Operator
+from .symbol import Symbol
+from .token_mapper import TokenMapper
+from .string_walker import StringWalker
+from .trie import Trie
+
+__all__ = [
+    "Tokenizer",
+    "Token",
+    "TokenType",
+    "Keyword",
+    "Operator",
+    "Symbol",
+    "TokenMapper",
+    "StringWalker",
+    "Trie",
+]

package/flowquery-py/src/tokenization/keyword.py

@@ -0,0 +1,48 @@
+"""Keyword enumeration for FlowQuery tokenization."""
+
+from enum import Enum
+
+
+class Keyword(Enum):
+    """Enumeration of all keywords in FlowQuery."""
+    
+    RETURN = "RETURN"
+    MATCH = "MATCH"
+    WHERE = "WHERE"
+    CREATE = "CREATE"
+    VIRTUAL = "VIRTUAL"
+    MERGE = "MERGE"
+    DELETE = "DELETE"
+    DETACH = "DETACH"
+    SET = "SET"
+    REMOVE = "REMOVE"
+    FOREACH = "FOREACH"
+    WITH = "WITH"
+    CALL = "CALL"
+    YIELD = "YIELD"
+    LOAD = "LOAD"
+    HEADERS = "HEADERS"
+    POST = "POST"
+    FROM = "FROM"
+    CSV = "CSV"
+    JSON = "JSON"
+    TEXT = "TEXT"
+    AS = "AS"
+    UNWIND = "UNWIND"
+    SUM = "SUM"
+    COLLECT = "COLLECT"
+    DISTINCT = "DISTINCT"
+    ORDER = "ORDER"
+    BY = "BY"
+    ASC = "ASC"
+    DESC = "DESC"
+    SKIP = "SKIP"
+    LIMIT = "LIMIT"
+    EOF = "EOF"
+    CASE = "CASE"
+    WHEN = "WHEN"
+    THEN = "THEN"
+    ELSE = "ELSE"
+    END = "END"
+    NULL = "NULL"
+    IN = "IN"

package/flowquery-py/src/tokenization/operator.py

@@ -0,0 +1,29 @@
+"""Operator enumeration for FlowQuery tokenization."""
+
+from enum import Enum
+
+
+class Operator(Enum):
+    """Enumeration of all operators in FlowQuery."""
+    
+    # Arithmetic
+    ADD = "+"
+    SUBTRACT = "-"
+    MULTIPLY = "*"
+    DIVIDE = "/"
+    MODULO = "%"
+    EXPONENT = "^"
+    # Comparison
+    EQUALS = "="
+    NOT_EQUALS = "<>"
+    LESS_THAN = "<"
+    LESS_THAN_OR_EQUAL = "<="
+    GREATER_THAN = ">"
+    GREATER_THAN_OR_EQUAL = ">="
+    IS = "IS"
+    # Logical
+    AND = "AND"
+    OR = "OR"
+    NOT = "NOT"
+    IN = "IN"
+    PIPE = "|"

package/flowquery-py/src/tokenization/string_walker.py

@@ -0,0 +1,158 @@
+"""Utility class for walking through a string character by character during tokenization."""
+
+from ..utils.string_utils import StringUtils
+
+
+class StringWalker:
+    """Utility class for walking through a string character by character during tokenization.
+    
+    Provides methods to check for specific character patterns, move through the string,
+    and extract substrings. Used by the Tokenizer to process input text.
+    
+    Example:
+        walker = StringWalker("WITH x as variable")
+        while not walker.is_at_end:
+            # Process characters
+    """
+
+    def __init__(self, text: str):
+        """Creates a new StringWalker for the given text.
+        
+        Args:
+            text: The input text to walk through
+        """
+        self._text = text
+        self._position = 0
+
+    @property
+    def position(self) -> int:
+        return self._position
+
+    @property
+    def current_char(self) -> str:
+        if self._position >= len(self._text):
+            return ''
+        return self._text[self._position]
+
+    @property
+    def next_char(self) -> str:
+        if self._position + 1 >= len(self._text):
+            return ''
+        return self._text[self._position + 1]
+
+    @property
+    def previous_char(self) -> str:
+        if self._position - 1 < 0:
+            return ''
+        return self._text[self._position - 1]
+
+    @property
+    def is_at_end(self) -> bool:
+        return self._position >= len(self._text)
+
+    def get_string(self, start_position: int) -> str:
+        return self._text[start_position:self._position]
+
+    def get_remaining_string(self) -> str:
+        return self._text[self._position:]
+
+    def check_for_single_comment(self) -> bool:
+        if self.single_line_comment_start():
+            while not self.is_at_end and not self.new_line():
+                self._position += 1
+            return True
+        return False
+
+    def check_for_multi_line_comment(self) -> bool:
+        if self.multi_line_comment_start():
+            while not self.is_at_end:
+                if self.multi_line_comment_end():
+                    self._position += 2
+                    return True
+                self._position += 1
+            raise ValueError(f"Unterminated multi-line comment at position {self._position}")
+        return False
+
+    def single_line_comment_start(self) -> bool:
+        return self.current_char == '/' and self.next_char == '/'
+
+    def multi_line_comment_start(self) -> bool:
+        return self.current_char == '/' and self.next_char == '*'
+
+    def multi_line_comment_end(self) -> bool:
+        return self.current_char == '*' and self.next_char == '/'
+
+    def new_line(self) -> bool:
+        return self.current_char == '\n'
+
+    def escaped(self, char: str) -> bool:
+        return self.current_char == '\\' and self.next_char == char
+
+    def escaped_brace(self) -> bool:
+        return ((self.current_char == '{' and self.next_char == '{') or 
+                (self.current_char == '}' and self.next_char == '}'))
+
+    def opening_brace(self) -> bool:
+        return self.current_char == '{'
+
+    def closing_brace(self) -> bool:
+        return self.current_char == '}'
+
+    def check_for_under_score(self) -> bool:
+        found_under_score = self.current_char == '_'
+        if found_under_score:
+            self._position += 1
+        return found_under_score
+
+    def check_for_letter(self) -> bool:
+        found_letter = self.current_char.lower() in StringUtils.letters
+        if found_letter:
+            self._position += 1
+        return found_letter
+
+    def check_for_digit(self) -> bool:
+        found_digit = self.current_char in StringUtils.digits
+        if found_digit:
+            self._position += 1
+        return found_digit
+
+    def check_for_quote(self) -> str | None:
+        quote_char = self.current_char
+        if quote_char in ('"', "'", '`'):
+            self._position += 1
+            return quote_char
+        return None
+
+    def check_for_string(self, value: str) -> bool:
+        _string = self._text[self._position:self._position + len(value)]
+        found_string = _string.lower() == value.lower()
+        if found_string:
+            self._position += len(value)
+        return found_string
+
+    def check_for_whitespace(self) -> bool:
+        return self.current_char in StringUtils.whitespace
+
+    def check_for_f_string_start(self) -> bool:
+        return self.current_char.lower() == 'f' and self.next_char in ("'", '"', '`')
+
+    def move_next(self) -> None:
+        self._position += 1
+
+    def move_by(self, steps: int) -> None:
+        self._position += steps
+
+    def move_previous(self) -> None:
+        self._position -= 1
+
+    def is_word(self, word: str | None) -> bool:
+        if word is None:
+            return False
+        return self._text[self._position:self._position + len(word)] == word
+
+    def word_continuation(self, word: str) -> bool:
+        next_pos = self._position + len(word)
+        if next_pos >= len(self._text):
+            return False
+        next_char = self._text[next_pos]
+        return next_char in StringUtils.word_valid_chars

package/flowquery-py/src/tokenization/symbol.py

@@ -0,0 +1,19 @@
+"""Symbol enumeration for FlowQuery tokenization."""
+
+from enum import Enum
+
+
+class Symbol(Enum):
+    """Enumeration of all symbols in FlowQuery."""
+    
+    LEFT_PARENTHESIS = "("
+    RIGHT_PARENTHESIS = ")"
+    COMMA = ","
+    DOT = "."
+    COLON = ":"
+    WHITESPACE = ""
+    OPENING_BRACE = "{"
+    CLOSING_BRACE = "}"
+    OPENING_BRACKET = "["
+    CLOSING_BRACKET = "]"
+    BACKTICK = "`"