flowquery 1.0.46 → 1.0.48

package/flowquery-py/src/tokenization/token_mapper.py

@@ -1,53 +0,0 @@
-"""Maps string values to tokens using a Trie for efficient lookup."""
-
-from enum import Enum
-from typing import Optional, Type
-
-from .token import Token
-from .trie import Trie
-
-
-class TokenMapper:
-    """Maps string values to tokens using a Trie for efficient lookup.
-
-    Takes an enum of keywords, operators, or symbols and builds a trie
-    for fast token matching during tokenization.
-
-    Example:
-        mapper = TokenMapper(Keyword)
-        token = mapper.map("WITH")
-    """
-
-    def __init__(self, enum_class: Type[Enum]) -> None:
-        """Creates a TokenMapper from an enum of token values.
-
-        Args:
-            enum_class: An enum class containing token values
-        """
-        self._trie = Trie()
-        self._enum = enum_class
-
-        for member in enum_class:
-            token = Token.method(member.name)
-            if token is not None and token.value is not None:
-                self._trie.insert(token)
-
-    def map(self, value: str) -> Optional[Token]:
-        """Maps a string value to its corresponding token.
-
-        Args:
-            value: The string value to map
-
-        Returns:
-            The matched token, or None if no match found
-        """
-        return self._trie.find(value)
-
-    @property
-    def last_found(self) -> Optional[str]:
-        """Gets the last matched string from the most recent map operation.
-
-        Returns:
-            The last found string, or None if no match
-        """
-        return self._trie.last_found

package/flowquery-py/src/tokenization/token_type.py

@@ -1,21 +0,0 @@
-"""Token type enumeration for FlowQuery tokenization."""
-
-from enum import Enum
-
-
-class TokenType(Enum):
-    """Enumeration of all token types in FlowQuery."""
-
-    KEYWORD = "KEYWORD"
-    BOOLEAN = "BOOLEAN"
-    OPERATOR = "OPERATOR"
-    UNARY_OPERATOR = "UNARY_OPERATOR"
-    IDENTIFIER = "IDENTIFIER"
-    STRING = "STRING"
-    F_STRING = "F-STRING"
-    BACKTICK_STRING = "BACKTICK_STRING"
-    NUMBER = "NUMBER"
-    SYMBOL = "SYMBOL"
-    WHITESPACE = "WHITESPACE"
-    COMMENT = "COMMENT"
-    EOF = "EOF"

package/flowquery-py/src/tokenization/tokenizer.py

@@ -1,214 +0,0 @@
-"""Tokenizes FlowQuery input strings into a sequence of tokens."""
-
-from typing import Callable, Iterator, List, Optional
-
-from ..utils.string_utils import StringUtils
-from .keyword import Keyword
-from .operator import Operator
-from .string_walker import StringWalker
-from .symbol import Symbol
-from .token import Token
-from .token_mapper import TokenMapper
-
-
-class Tokenizer:
-    """Tokenizes FlowQuery input strings into a sequence of tokens.
-
-    The tokenizer performs lexical analysis, breaking down the input text into
-    meaningful tokens such as keywords, identifiers, operators, strings, numbers,
-    and symbols. It handles comments, whitespace, and f-strings.
-
-    Example:
-        tokenizer = Tokenizer("WITH x = 1 RETURN x")
-        tokens = tokenizer.tokenize()
-    """
-
-    def __init__(self, input_: str):
-        """Creates a new Tokenizer instance for the given input.
-
-        Args:
-            input_: The FlowQuery input string to tokenize
-        """
-        self._walker = StringWalker(input_)
-        self._keywords = TokenMapper(Keyword)
-        self._symbols = TokenMapper(Symbol)
-        self._operators = TokenMapper(Operator)
-
-    def tokenize(self) -> List[Token]:
-        """Tokenizes the input string into an array of tokens.
-
-        Returns:
-            An array of Token objects representing the tokenized input
-
-        Raises:
-            ValueError: If an unrecognized token is encountered
-        """
-        tokens: List[Token] = []
-        last: Optional[Token] = None
-
-        while not self._walker.is_at_end:
-            tokens.extend(self._f_string())
-            last = self._get_last_non_whitespace_or_non_comment_token(tokens) or last
-            token = self._get_next_token(last)
-            if token is None:
-                raise ValueError(f"Unrecognized token at position {self._walker.position}")
-            token.position = self._walker.position
-            tokens.append(token)
-
-        return tokens
-
-    def _get_last_non_whitespace_or_non_comment_token(self, tokens: List[Token]) -> Optional[Token]:
-        if len(tokens) == 0:
-            return None
-        if not tokens[-1].is_whitespace_or_comment():
-            return tokens[-1]
-        return None
-
-    def _get_next_token(self, last: Optional[Token] = None) -> Optional[Token]:
-        if self._walker.is_at_end:
-            return Token.EOF()
-        return (
-            self._comment() or
-            self._whitespace() or
-            self._lookup(self._keywords) or
-            self._lookup(self._operators, last, self._skip_minus) or
-            self._boolean() or
-            self._identifier() or
-            self._string() or
-            self._number() or
-            self._lookup(self._symbols)
-        )
-
-    def _comment(self) -> Optional[Token]:
-        start_position = self._walker.position
-        if self._walker.check_for_single_comment() or self._walker.check_for_multi_line_comment():
-            uncommented = StringUtils.uncomment(self._walker.get_string(start_position))
-            return Token.COMMENT(uncommented)
-        return None
-
-    def _boolean(self) -> Optional[Token]:
-        start_position = self._walker.position
-        if self._walker.check_for_string("TRUE"):
-            return Token.BOOLEAN(self._walker.get_string(start_position).upper())
-        if self._walker.check_for_string("FALSE"):
-            return Token.BOOLEAN(self._walker.get_string(start_position).upper())
-        return None
-
-    def _identifier(self) -> Optional[Token]:
-        start_position = self._walker.position
-        if self._walker.check_for_under_score() or self._walker.check_for_letter():
-            while (not self._walker.is_at_end and
-                   (self._walker.check_for_letter() or
-                    self._walker.check_for_digit() or
-                    self._walker.check_for_under_score())):
-                pass
-            return Token.IDENTIFIER(self._walker.get_string(start_position))
-        return None
-
-    def _string(self) -> Optional[Token]:
-        start_position = self._walker.position
-        quote_char = self._walker.check_for_quote()
-        if quote_char is None:
-            return None
-
-        while not self._walker.is_at_end:
-            if self._walker.escaped(quote_char):
-                self._walker.move_next()
-                self._walker.move_next()
-                continue
-            if self._walker.check_for_string(quote_char):
-                value = self._walker.get_string(start_position)
-                if quote_char == Symbol.BACKTICK.value:
-                    return Token.BACKTICK_STRING(value, quote_char)
-                return Token.STRING(value, quote_char)
-            self._walker.move_next()
-
-        raise ValueError(f"Unterminated string at position {start_position}")
-
-    def _f_string(self) -> Iterator[Token]:
-        if not self._walker.check_for_f_string_start():
-            return
-
-        self._walker.move_next()  # skip the f
-        position = self._walker.position
-        quote_char = self._walker.check_for_quote()
-        if quote_char is None:
-            return
-
-        while not self._walker.is_at_end:
-            if self._walker.escaped(quote_char) or self._walker.escaped_brace():
-                self._walker.move_next()
-                self._walker.move_next()
-                continue
-
-            if self._walker.opening_brace():
-                yield Token.F_STRING(self._walker.get_string(position), quote_char)
-                position = self._walker.position
-                yield Token.OPENING_BRACE()
-                self._walker.move_next()  # skip the opening brace
-                position = self._walker.position
-
-                while not self._walker.is_at_end and not self._walker.closing_brace():
-                    token = self._get_next_token()
-                    if token is not None:
-                        yield token
-                    else:
-                        break
-                    if self._walker.closing_brace():
-                        yield Token.CLOSING_BRACE()
-                        self._walker.move_next()  # skip the closing brace
-                        position = self._walker.position
-                        break
-
-            if self._walker.check_for_string(quote_char):
-                yield Token.F_STRING(self._walker.get_string(position), quote_char)
-                return
-
-            self._walker.move_next()
-
-    def _whitespace(self) -> Optional[Token]:
-        found_whitespace = False
-        while not self._walker.is_at_end and self._walker.check_for_whitespace():
-            self._walker.move_next()
-            found_whitespace = True
-        return Token.WHITESPACE() if found_whitespace else None
-
-    def _number(self) -> Optional[Token]:
-        start_position = self._walker.position
-        if self._walker.check_for_string("-") or self._walker.check_for_digit():
-            while not self._walker.is_at_end and self._walker.check_for_digit():
-                pass
-            if self._walker.check_for_string(Symbol.DOT.value):
-                decimal_digits = 0
-                while not self._walker.is_at_end and self._walker.check_for_digit():
-                    decimal_digits += 1
-                if decimal_digits == 0:
-                    self._walker.move_previous()
-            number_str = self._walker.get_string(start_position)
-            return Token.NUMBER(number_str)
-        return None
-
-    def _lookup(
-        self,
-        mapper: TokenMapper,
-        last: Optional[Token] = None,
-        skip: Optional[Callable[[Optional[Token], Token], bool]] = None
-    ) -> Optional[Token]:
-        token = mapper.map(self._walker.get_remaining_string())
-        if token is not None and token.value is not None:
-            if token.can_be_identifier and self._walker.word_continuation(token.value):
-                return None
-            if skip and last and skip(last, token):
-                return None
-            self._walker.move_by(len(token.value))
-            if mapper.last_found is not None:
-                token.case_sensitive_value = mapper.last_found
-            return token
-        return None
-
-    def _skip_minus(self, last: Optional[Token], current: Token) -> bool:
-        if last is None:
-            return False
-        if (last.is_keyword() or last.is_comma() or last.is_colon()) and current.is_negation():
-            return True
-        return False

package/flowquery-py/src/tokenization/trie.py

@@ -1,125 +0,0 @@
-"""Trie (prefix tree) data structure for efficient keyword and operator lookup."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Optional
-
-if TYPE_CHECKING:
-    from .token import Token
-
-
-class TrieNode:
-    """Represents a node in a Trie data structure.
-
-    Each node can have children nodes (one per character) and may contain a token
-    if the path to this node represents a complete word.
-    """
-
-    def __init__(self) -> None:
-        self._children: dict[str, TrieNode] = {}
-        self._token: Optional[Token] = None
-
-    def map(self, char: str) -> TrieNode:
-        if char not in self._children:
-            self._children[char] = TrieNode()
-        return self._children[char]
-
-    def retrieve(self, char: str) -> Optional[TrieNode]:
-        return self._children.get(char)
-
-    @property
-    def token(self) -> Optional[Token]:
-        return self._token
-
-    @token.setter
-    def token(self, token: Token) -> None:
-        self._token = token
-
-    def is_end_of_word(self) -> bool:
-        return self._token is not None
-
-    def no_children(self) -> bool:
-        return len(self._children) == 0
-
-
-class Trie:
-    """Trie (prefix tree) data structure for efficient keyword and operator lookup.
-
-    Used during tokenization to quickly match input strings against known keywords
-    and operators. Supports case-insensitive matching and tracks the longest match found.
-
-    Example:
-        trie = Trie()
-        trie.insert(Token.WITH)
-        found = trie.find("WITH")
-    """
-
-    def __init__(self) -> None:
-        self._root = TrieNode()
-        self._max_length = 0
-        self._last_found: Optional[str] = None
-
-    def insert(self, token: Token) -> None:
-        """Inserts a token into the trie.
-
-        Args:
-            token: The token to insert
-
-        Raises:
-            ValueError: If the token value is None or empty
-        """
-        if token.value is None or len(token.value) == 0:
-            raise ValueError("Token value cannot be null or empty")
-
-        current_node = self._root
-        for char in token.value:
-            current_node = current_node.map(char.lower())
-
-        if len(token.value) > self._max_length:
-            self._max_length = len(token.value)
-
-        current_node.token = token
-
-    def find(self, value: str) -> Optional[Token]:
-        """Finds a token by searching for the longest matching prefix in the trie.
-
-        Args:
-            value: The string value to search for
-
-        Returns:
-            The token if found, None otherwise
-        """
-        if len(value) == 0:
-            return None
-
-        index = 0
-        current: Optional[TrieNode] = None
-        found: Optional[Token] = None
-        self._last_found = None
-
-        while True:
-            next_node = (current or self._root).retrieve(value[index].lower())
-            if next_node is None:
-                break
-            current = next_node
-            if current.is_end_of_word():
-                found = current.token
-                self._last_found = value[:index + 1]
-            index += 1
-            if index >= len(value) or index > self._max_length:
-                break
-
-        if current is not None and current.is_end_of_word():
-            found = current.token
-            self._last_found = value[:index]
-
-        return found
-
-    @property
-    def last_found(self) -> Optional[str]:
-        """Gets the last matched string from the most recent find operation.
-
-        Returns:
-            The last found string, or None if no match was found
-        """
-        return self._last_found

package/flowquery-py/src/utils/__init__.py

@@ -1,6 +0,0 @@
-"""Utils module for FlowQuery."""
-
-from .object_utils import ObjectUtils
-from .string_utils import StringUtils
-
-__all__ = ["StringUtils", "ObjectUtils"]

package/flowquery-py/src/utils/object_utils.py

@@ -1,20 +0,0 @@
-"""Utility class for object-related operations."""
-
-from typing import Any, List, Type
-
-
-class ObjectUtils:
-    """Utility class for object-related operations."""
-
-    @staticmethod
-    def is_instance_of_any(obj: Any, classes: List[Type[Any]]) -> bool:
-        """Checks if an object is an instance of any of the provided classes.
-
-        Args:
-            obj: The object to check
-            classes: Array of class constructors to test against
-
-        Returns:
-            True if the object is an instance of any class, False otherwise
-        """
-        return any(isinstance(obj, cls) for cls in classes)

package/flowquery-py/src/utils/string_utils.py

@@ -1,113 +0,0 @@
-"""Utility class for string manipulation and validation."""
-
-
-class StringUtils:
-    """Utility class for string manipulation and validation.
-
-    Provides methods for handling quoted strings, comments, escape sequences,
-    and identifier validation.
-    """
-
-    quotes = ['"', "'", '`']
-    letters = 'abcdefghijklmnopqrstuvwxyz'
-    digits = '0123456789'
-    whitespace = ' \t\n\r'
-    word_valid_chars = letters + letters.upper() + digits + '_'
-
-    @staticmethod
-    def unquote(s: str) -> str:
-        """Removes surrounding quotes from a string.
-
-        Args:
-            s: The string to unquote
-
-        Returns:
-            The unquoted string
-        """
-        if len(s) == 0:
-            return s
-        if len(s) == 1 and s in StringUtils.quotes:
-            return ''
-        first = s[0]
-        last = s[-1]
-        if first in StringUtils.quotes and first == last:
-            return s[1:-1]
-        if last in StringUtils.quotes and first != last:
-            return s[:-1]
-        if first in StringUtils.quotes and first != last:
-            return s[1:]
-        return s
-
-    @staticmethod
-    def uncomment(s: str) -> str:
-        """Removes comment markers from a string.
-
-        Args:
-            s: The comment string
-
-        Returns:
-            The string without comment markers
-        """
-        if len(s) < 2:
-            return s
-        if s[0] == '/' and s[1] == '/':
-            return s[2:]
-        if s[0] == '/' and s[1] == '*' and s[-2] == '*' and s[-1] == '/':
-            return s[2:-2]
-        return s
-
-    @staticmethod
-    def remove_escaped_quotes(s: str, quote_char: str) -> str:
-        """Removes escape sequences before quotes in a string.
-
-        Args:
-            s: The string to process
-            quote_char: The quote character that was escaped
-
-        Returns:
-            The string with escape sequences removed
-        """
-        unescaped = ''
-        i = 0
-        while i < len(s):
-            if i < len(s) - 1 and s[i] == '\\' and s[i + 1] == quote_char:
-                i += 1
-            unescaped += s[i]
-            i += 1
-        return unescaped
-
-    @staticmethod
-    def remove_escaped_braces(s: str) -> str:
-        """Removes escaped braces ({{ and }}) from f-strings.
-
-        Args:
-            s: The string to process
-
-        Returns:
-            The string with escaped braces resolved
-        """
-        unescaped = ''
-        i = 0
-        while i < len(s):
-            if i < len(s) - 1 and ((s[i] == '{' and s[i + 1] == '{') or (s[i] == '}' and s[i + 1] == '}')):
-                i += 1
-            unescaped += s[i]
-            i += 1
-        return unescaped
-
-    @staticmethod
-    def can_be_identifier(s: str) -> bool:
-        """Checks if a string is a valid identifier.
-
-        Args:
-            s: The string to validate
-
-        Returns:
-            True if the string can be used as an identifier, false otherwise
-        """
-        lower = s.lower()
-        if len(lower) == 0:
-            return False
-        if lower[0] not in StringUtils.letters and lower[0] != '_':
-            return False
-        return all(char in StringUtils.word_valid_chars for char in lower)

package/flowquery-py/tests/__init__.py

@@ -1 +0,0 @@
-"""Tests package for FlowQuery."""

package/flowquery-py/tests/compute/__init__.py

@@ -1 +0,0 @@
-"""Compute tests package."""