flowquery 1.0.16 → 1.0.17

package/flowquery-py/src/tokenization/token_type.py

@@ -0,0 +1,21 @@
+"""Token type enumeration for FlowQuery tokenization."""
+
+from enum import Enum
+
+
+class TokenType(Enum):
+    """Enumeration of all token types in FlowQuery."""
+    
+    KEYWORD = "KEYWORD"
+    BOOLEAN = "BOOLEAN"
+    OPERATOR = "OPERATOR"
+    UNARY_OPERATOR = "UNARY_OPERATOR"
+    IDENTIFIER = "IDENTIFIER"
+    STRING = "STRING"
+    F_STRING = "F-STRING"
+    BACKTICK_STRING = "BACKTICK_STRING"
+    NUMBER = "NUMBER"
+    SYMBOL = "SYMBOL"
+    WHITESPACE = "WHITESPACE"
+    COMMENT = "COMMENT"
+    EOF = "EOF"

package/flowquery-py/src/tokenization/tokenizer.py

@@ -0,0 +1,214 @@
+"""Tokenizes FlowQuery input strings into a sequence of tokens."""
+
+from typing import List, Optional, Iterator, Callable
+
+from ..utils.string_utils import StringUtils
+from .keyword import Keyword
+from .operator import Operator
+from .string_walker import StringWalker
+from .symbol import Symbol
+from .token import Token
+from .token_mapper import TokenMapper
+
+
+class Tokenizer:
+    """Tokenizes FlowQuery input strings into a sequence of tokens.
+    
+    The tokenizer performs lexical analysis, breaking down the input text into
+    meaningful tokens such as keywords, identifiers, operators, strings, numbers,
+    and symbols. It handles comments, whitespace, and f-strings.
+    
+    Example:
+        tokenizer = Tokenizer("WITH x = 1 RETURN x")
+        tokens = tokenizer.tokenize()
+    """
+
+    def __init__(self, input_: str):
+        """Creates a new Tokenizer instance for the given input.
+        
+        Args:
+            input_: The FlowQuery input string to tokenize
+        """
+        self._walker = StringWalker(input_)
+        self._keywords = TokenMapper(Keyword)
+        self._symbols = TokenMapper(Symbol)
+        self._operators = TokenMapper(Operator)
+
+    def tokenize(self) -> List[Token]:
+        """Tokenizes the input string into an array of tokens.
+        
+        Returns:
+            An array of Token objects representing the tokenized input
+            
+        Raises:
+            ValueError: If an unrecognized token is encountered
+        """
+        tokens: List[Token] = []
+        last: Optional[Token] = None
+        
+        while not self._walker.is_at_end:
+            tokens.extend(self._f_string())
+            last = self._get_last_non_whitespace_or_non_comment_token(tokens) or last
+            token = self._get_next_token(last)
+            if token is None:
+                raise ValueError(f"Unrecognized token at position {self._walker.position}")
+            token.position = self._walker.position
+            tokens.append(token)
+        
+        return tokens
+
+    def _get_last_non_whitespace_or_non_comment_token(self, tokens: List[Token]) -> Optional[Token]:
+        if len(tokens) == 0:
+            return None
+        if not tokens[-1].is_whitespace_or_comment():
+            return tokens[-1]
+        return None
+
+    def _get_next_token(self, last: Optional[Token] = None) -> Optional[Token]:
+        if self._walker.is_at_end:
+            return Token.EOF
+        return (
+            self._comment() or
+            self._whitespace() or
+            self._lookup(self._keywords) or
+            self._lookup(self._operators, last, self._skip_minus) or
+            self._boolean() or
+            self._identifier() or
+            self._string() or
+            self._number() or
+            self._lookup(self._symbols)
+        )
+
+    def _comment(self) -> Optional[Token]:
+        start_position = self._walker.position
+        if self._walker.check_for_single_comment() or self._walker.check_for_multi_line_comment():
+            uncommented = StringUtils.uncomment(self._walker.get_string(start_position))
+            return Token.COMMENT(uncommented)
+        return None
+
+    def _boolean(self) -> Optional[Token]:
+        start_position = self._walker.position
+        if self._walker.check_for_string("TRUE"):
+            return Token.BOOLEAN(self._walker.get_string(start_position).upper())
+        if self._walker.check_for_string("FALSE"):
+            return Token.BOOLEAN(self._walker.get_string(start_position).upper())
+        return None
+
+    def _identifier(self) -> Optional[Token]:
+        start_position = self._walker.position
+        if self._walker.check_for_under_score() or self._walker.check_for_letter():
+            while (not self._walker.is_at_end and 
+                   (self._walker.check_for_letter() or 
+                    self._walker.check_for_digit() or 
+                    self._walker.check_for_under_score())):
+                pass
+            return Token.IDENTIFIER(self._walker.get_string(start_position))
+        return None
+
+    def _string(self) -> Optional[Token]:
+        start_position = self._walker.position
+        quote_char = self._walker.check_for_quote()
+        if quote_char is None:
+            return None
+        
+        while not self._walker.is_at_end:
+            if self._walker.escaped(quote_char):
+                self._walker.move_next()
+                self._walker.move_next()
+                continue
+            if self._walker.check_for_string(quote_char):
+                value = self._walker.get_string(start_position)
+                if quote_char == Symbol.BACKTICK.value:
+                    return Token.BACKTICK_STRING(value, quote_char)
+                return Token.STRING(value, quote_char)
+            self._walker.move_next()
+        
+        raise ValueError(f"Unterminated string at position {start_position}")
+
+    def _f_string(self) -> Iterator[Token]:
+        if not self._walker.check_for_f_string_start():
+            return
+        
+        self._walker.move_next()  # skip the f
+        position = self._walker.position
+        quote_char = self._walker.check_for_quote()
+        if quote_char is None:
+            return
+        
+        while not self._walker.is_at_end:
+            if self._walker.escaped(quote_char) or self._walker.escaped_brace():
+                self._walker.move_next()
+                self._walker.move_next()
+                continue
+            
+            if self._walker.opening_brace():
+                yield Token.F_STRING(self._walker.get_string(position), quote_char)
+                position = self._walker.position
+                yield Token.OPENING_BRACE
+                self._walker.move_next()  # skip the opening brace
+                position = self._walker.position
+                
+                while not self._walker.is_at_end and not self._walker.closing_brace():
+                    token = self._get_next_token()
+                    if token is not None:
+                        yield token
+                    else:
+                        break
+                    if self._walker.closing_brace():
+                        yield Token.CLOSING_BRACE
+                        self._walker.move_next()  # skip the closing brace
+                        position = self._walker.position
+                        break
+            
+            if self._walker.check_for_string(quote_char):
+                yield Token.F_STRING(self._walker.get_string(position), quote_char)
+                return
+            
+            self._walker.move_next()
+
+    def _whitespace(self) -> Optional[Token]:
+        found_whitespace = False
+        while not self._walker.is_at_end and self._walker.check_for_whitespace():
+            self._walker.move_next()
+            found_whitespace = True
+        return Token.WHITESPACE if found_whitespace else None
+
+    def _number(self) -> Optional[Token]:
+        start_position = self._walker.position
+        if self._walker.check_for_string("-") or self._walker.check_for_digit():
+            while not self._walker.is_at_end and self._walker.check_for_digit():
+                pass
+            if self._walker.check_for_string(Symbol.DOT.value):
+                decimal_digits = 0
+                while not self._walker.is_at_end and self._walker.check_for_digit():
+                    decimal_digits += 1
+                if decimal_digits == 0:
+                    self._walker.move_previous()
+            number_str = self._walker.get_string(start_position)
+            return Token.NUMBER(number_str)
+        return None
+
+    def _lookup(
+        self,
+        mapper: TokenMapper,
+        last: Optional[Token] = None,
+        skip: Optional[Callable[[Optional[Token], Token], bool]] = None
+    ) -> Optional[Token]:
+        token = mapper.map(self._walker.get_remaining_string())
+        if token is not None and token.value is not None:
+            if token.can_be_identifier and self._walker.word_continuation(token.value):
+                return None
+            if skip and last and skip(last, token):
+                return None
+            self._walker.move_by(len(token.value))
+            if mapper.last_found is not None:
+                token.case_sensitive_value = mapper.last_found
+            return token
+        return None
+
+    def _skip_minus(self, last: Optional[Token], current: Token) -> bool:
+        if last is None:
+            return False
+        if (last.is_keyword() or last.is_comma() or last.is_colon()) and current.is_negation():
+            return True
+        return False

package/flowquery-py/src/tokenization/trie.py

@@ -0,0 +1,124 @@
+"""Trie (prefix tree) data structure for efficient keyword and operator lookup."""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from .token import Token
+
+
+class TrieNode:
+    """Represents a node in a Trie data structure.
+    
+    Each node can have children nodes (one per character) and may contain a token
+    if the path to this node represents a complete word.
+    """
+
+    def __init__(self):
+        self._children: dict[str, TrieNode] = {}
+        self._token: Optional[Token] = None
+
+    def map(self, char: str) -> TrieNode:
+        if char not in self._children:
+            self._children[char] = TrieNode()
+        return self._children[char]
+
+    def retrieve(self, char: str) -> Optional[TrieNode]:
+        return self._children.get(char)
+
+    @property
+    def token(self) -> Optional[Token]:
+        return self._token
+
+    @token.setter
+    def token(self, token: Token) -> None:
+        self._token = token
+
+    def is_end_of_word(self) -> bool:
+        return self._token is not None
+
+    def no_children(self) -> bool:
+        return len(self._children) == 0
+
+
+class Trie:
+    """Trie (prefix tree) data structure for efficient keyword and operator lookup.
+    
+    Used during tokenization to quickly match input strings against known keywords
+    and operators. Supports case-insensitive matching and tracks the longest match found.
+    
+    Example:
+        trie = Trie()
+        trie.insert(Token.WITH)
+        found = trie.find("WITH")
+    """
+
+    def __init__(self):
+        self._root = TrieNode()
+        self._max_length = 0
+        self._last_found: Optional[str] = None
+
+    def insert(self, token: Token) -> None:
+        """Inserts a token into the trie.
+        
+        Args:
+            token: The token to insert
+            
+        Raises:
+            ValueError: If the token value is None or empty
+        """
+        if token.value is None or len(token.value) == 0:
+            raise ValueError("Token value cannot be null or empty")
+        
+        current_node = self._root
+        for char in token.value:
+            current_node = current_node.map(char.lower())
+        
+        if len(token.value) > self._max_length:
+            self._max_length = len(token.value)
+        
+        current_node.token = token
+
+    def find(self, value: str) -> Optional[Token]:
+        """Finds a token by searching for the longest matching prefix in the trie.
+        
+        Args:
+            value: The string value to search for
+            
+        Returns:
+            The token if found, None otherwise
+        """
+        if len(value) == 0:
+            return None
+        
+        index = 0
+        current: Optional[TrieNode] = None
+        found: Optional[Token] = None
+        self._last_found = None
+        
+        while True:
+            next_node = (current or self._root).retrieve(value[index].lower())
+            if next_node is None:
+                break
+            current = next_node
+            if current.is_end_of_word():
+                found = current.token
+                self._last_found = value[:index + 1]
+            index += 1
+            if index >= len(value) or index > self._max_length:
+                break
+        
+        if current is not None and current.is_end_of_word():
+            found = current.token
+            self._last_found = value[:index]
+        
+        return found
+
+    @property
+    def last_found(self) -> Optional[str]:
+        """Gets the last matched string from the most recent find operation.
+        
+        Returns:
+            The last found string, or None if no match was found
+        """
+        return self._last_found

package/flowquery-py/src/utils/__init__.py

@@ -0,0 +1,6 @@
+"""Utils module for FlowQuery."""
+
+from .string_utils import StringUtils
+from .object_utils import ObjectUtils
+
+__all__ = ["StringUtils", "ObjectUtils"]

package/flowquery-py/src/utils/object_utils.py

@@ -0,0 +1,20 @@
+"""Utility class for object-related operations."""
+
+from typing import Any, List, Type
+
+
+class ObjectUtils:
+    """Utility class for object-related operations."""
+
+    @staticmethod
+    def is_instance_of_any(obj: Any, classes: List[Type]) -> bool:
+        """Checks if an object is an instance of any of the provided classes.
+        
+        Args:
+            obj: The object to check
+            classes: Array of class constructors to test against
+            
+        Returns:
+            True if the object is an instance of any class, False otherwise
+        """
+        return any(isinstance(obj, cls) for cls in classes)

package/flowquery-py/src/utils/string_utils.py

@@ -0,0 +1,113 @@
+"""Utility class for string manipulation and validation."""
+
+
+class StringUtils:
+    """Utility class for string manipulation and validation.
+    
+    Provides methods for handling quoted strings, comments, escape sequences,
+    and identifier validation.
+    """
+    
+    quotes = ['"', "'", '`']
+    letters = 'abcdefghijklmnopqrstuvwxyz'
+    digits = '0123456789'
+    whitespace = ' \t\n\r'
+    word_valid_chars = letters + letters.upper() + digits + '_'
+
+    @staticmethod
+    def unquote(s: str) -> str:
+        """Removes surrounding quotes from a string.
+        
+        Args:
+            s: The string to unquote
+            
+        Returns:
+            The unquoted string
+        """
+        if len(s) == 0:
+            return s
+        if len(s) == 1 and s in StringUtils.quotes:
+            return ''
+        first = s[0]
+        last = s[-1]
+        if first in StringUtils.quotes and first == last:
+            return s[1:-1]
+        if last in StringUtils.quotes and first != last:
+            return s[:-1]
+        if first in StringUtils.quotes and first != last:
+            return s[1:]
+        return s
+
+    @staticmethod
+    def uncomment(s: str) -> str:
+        """Removes comment markers from a string.
+        
+        Args:
+            s: The comment string
+            
+        Returns:
+            The string without comment markers
+        """
+        if len(s) < 2:
+            return s
+        if s[0] == '/' and s[1] == '/':
+            return s[2:]
+        if s[0] == '/' and s[1] == '*' and s[-2] == '*' and s[-1] == '/':
+            return s[2:-2]
+        return s
+
+    @staticmethod
+    def remove_escaped_quotes(s: str, quote_char: str) -> str:
+        """Removes escape sequences before quotes in a string.
+        
+        Args:
+            s: The string to process
+            quote_char: The quote character that was escaped
+            
+        Returns:
+            The string with escape sequences removed
+        """
+        unescaped = ''
+        i = 0
+        while i < len(s):
+            if i < len(s) - 1 and s[i] == '\\' and s[i + 1] == quote_char:
+                i += 1
+            unescaped += s[i]
+            i += 1
+        return unescaped
+
+    @staticmethod
+    def remove_escaped_braces(s: str) -> str:
+        """Removes escaped braces ({{ and }}) from f-strings.
+        
+        Args:
+            s: The string to process
+            
+        Returns:
+            The string with escaped braces resolved
+        """
+        unescaped = ''
+        i = 0
+        while i < len(s):
+            if i < len(s) - 1 and ((s[i] == '{' and s[i + 1] == '{') or (s[i] == '}' and s[i + 1] == '}')):
+                i += 1
+            unescaped += s[i]
+            i += 1
+        return unescaped
+
+    @staticmethod
+    def can_be_identifier(s: str) -> bool:
+        """Checks if a string is a valid identifier.
+        
+        Args:
+            s: The string to validate
+            
+        Returns:
+            True if the string can be used as an identifier, false otherwise
+        """
+        lower = s.lower()
+        if len(lower) == 0:
+            return False
+        if lower[0] not in StringUtils.letters and lower[0] != '_':
+            return False
+        return all(char in StringUtils.word_valid_chars for char in lower)

package/flowquery-py/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests package for FlowQuery."""

package/flowquery-py/tests/compute/__init__.py

@@ -0,0 +1 @@
+"""Compute tests package."""