flowquery 1.0.32 → 1.0.33

package/flowquery-py/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowquery"
-version = "1.0.22"
+version = "1.0.23"
 description = "A declarative query language for data processing pipelines"
 readme = "README.md"
 requires-python = ">=3.10"

package/flowquery-py/src/__init__.py

@@ -6,6 +6,7 @@ This is the Python implementation of FlowQuery.
 This module provides the core components for defining, parsing, and executing FlowQuery queries.
 """
 
+from .compute.flowquery import FlowQuery
 from .compute.runner import Runner
 from .io.command_line import CommandLine
 from .parsing.functions.aggregate_function import AggregateFunction
@@ -21,6 +22,7 @@ from .parsing.functions.reducer_element import ReducerElement
 from .parsing.parser import Parser
 
 __all__ = [
+    "FlowQuery",
     "Runner",
     "CommandLine",
     "Parser",

package/flowquery-py/src/compute/__init__.py

@@ -1,5 +1,6 @@
 """Compute module for FlowQuery."""
 
+from .flowquery import FlowQuery
 from .runner import Runner
 
-__all__ = ["Runner"]
+__all__ = ["FlowQuery", "Runner"]

package/flowquery-py/src/compute/flowquery.py

@@ -0,0 +1,68 @@
+"""FlowQuery public API surface.
+
+Extends Runner with extensibility features such as function listing
+and plugin registration, keeping the Runner focused on execution.
+"""
+
+from typing import List, Optional, Type
+
+from ..parsing.functions.function import Function
+from ..parsing.functions.function_factory import FunctionFactory
+from ..parsing.functions.function_metadata import (
+    FunctionMetadata,
+    get_function_metadata,
+)
+from .runner import Runner
+
+
+class FlowQuery(Runner):
+    """FlowQuery is the public API surface for the FlowQuery library.
+
+    It extends Runner with convenience class methods for function
+    introspection and plugin registration.
+
+    Example:
+        fq = FlowQuery("WITH 1 as x RETURN x")
+        await fq.run()
+        print(fq.results)  # [{'x': 1}]
+
+        # List all registered functions
+        functions = FlowQuery.list_functions()
+    """
+
+    #: Base Function class for creating custom plugin functions.
+    Function: Type[Function] = Function
+
+    @staticmethod
+    def list_functions(
+        category: Optional[str] = None,
+        async_only: bool = False,
+        sync_only: bool = False,
+    ) -> List[FunctionMetadata]:
+        """List all registered functions with their metadata.
+
+        Args:
+            category: Optional category filter
+            async_only: If True, return only async functions
+            sync_only: If True, return only sync functions
+
+        Returns:
+            List of function metadata
+        """
+        return FunctionFactory.list_functions(
+            category=category,
+            async_only=async_only,
+            sync_only=sync_only,
+        )
+
+    @staticmethod
+    def get_function_metadata(name: str) -> Optional[FunctionMetadata]:
+        """Get metadata for a specific function.
+
+        Args:
+            name: The function name
+
+        Returns:
+            Function metadata or None
+        """
+        return get_function_metadata(name.lower())

package/flowquery-py/src/graph/node.py

@@ -75,7 +75,7 @@ class Node(ASTNode):
             return bool(record[key] == expression.value())
         return True
 
-    def set_value(self, value: Dict[str, Any]) -> None:
+    def set_value(self, value: Optional[Dict[str, Any]]) -> None:
         self._value = value  # type: ignore[assignment]
 
     def value(self) -> Optional['NodeRecord']:

package/flowquery-py/src/parsing/operations/__init__.py

@@ -12,6 +12,8 @@ from .match import Match
 from .operation import Operation
 from .projection import Projection
 from .return_op import Return
+from .union import Union
+from .union_all import UnionAll
 from .unwind import Unwind
 from .where import Where
 from .with_op import With
@@ -32,4 +34,6 @@ __all__ = [
     "Match",
     "CreateNode",
     "CreateRelationship",
+    "Union",
+    "UnionAll",
 ]

package/flowquery-py/src/parsing/operations/match.py

@@ -2,6 +2,7 @@
 
 from typing import List, Optional
 
+from ...graph.node import Node
 from ...graph.pattern import Pattern
 from ...graph.patterns import Patterns
 from .operation import Operation
@@ -10,21 +11,42 @@ from .operation import Operation
 class Match(Operation):
     """Represents a MATCH operation for graph pattern matching."""
 
-    def __init__(self, patterns: Optional[List[Pattern]] = None) -> None:
+    def __init__(self, patterns: Optional[List[Pattern]] = None, optional: bool = False) -> None:
         super().__init__()
         self._patterns = Patterns(patterns or [])
+        self._optional = optional
 
     @property
     def patterns(self) -> List[Pattern]:
         return self._patterns.patterns if self._patterns else []
 
+    @property
+    def optional(self) -> bool:
+        return self._optional
+
+    def __str__(self) -> str:
+        return "OptionalMatch" if self._optional else "Match"
+
     async def run(self) -> None:
-        """Executes the match operation by chaining the patterns together."""
+        """Executes the match operation by chaining the patterns together.
+        If optional and no match is found, continues with null values."""
         await self._patterns.initialize()
+        matched = False
 
         async def to_do_next() -> None:
+            nonlocal matched
+            matched = True
             if self.next:
                 await self.next.run()
 
         self._patterns.to_do_next = to_do_next
         await self._patterns.traverse()
+
+        # For OPTIONAL MATCH: if nothing matched, continue with None values
+        if not matched and self._optional:
+            for pattern in self._patterns.patterns:
+                for element in pattern.chain:
+                    if isinstance(element, Node):
+                        element.set_value(None)
+            if self.next:
+                await self.next.run()

package/flowquery-py/src/parsing/operations/union.py

@@ -0,0 +1,115 @@
+"""Represents a UNION operation that combines results from two sub-queries."""
+
+import json
+from typing import Any, Dict, List, Optional
+
+from .operation import Operation
+
+
+class Union(Operation):
+    """Represents a UNION operation that combines results from two sub-queries.
+
+    UNION merges the results of a left and right query pipeline, removing
+    duplicate rows. Both sides must return the same column names.
+
+    Example:
+        WITH 1 AS x RETURN x
+        UNION
+        WITH 2 AS x RETURN x
+        # Results: [{x: 1}, {x: 2}]
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._left: Optional[Operation] = None
+        self._right: Optional[Operation] = None
+        self._results: List[Dict[str, Any]] = []
+
+    @property
+    def left(self) -> Operation:
+        if self._left is None:
+            raise ValueError("Left operation is not set")
+        return self._left
+
+    @left.setter
+    def left(self, operation: Operation) -> None:
+        self._left = operation
+
+    @property
+    def right(self) -> Operation:
+        if self._right is None:
+            raise ValueError("Right operation is not set")
+        return self._right
+
+    @right.setter
+    def right(self, operation: Operation) -> None:
+        self._right = operation
+
+    @staticmethod
+    def _last_in_chain(operation: Operation) -> Operation:
+        current = operation
+        while current.next is not None:
+            current = current.next
+        return current
+
+    async def initialize(self) -> None:
+        self._results = []
+        if self.next:
+            await self.next.initialize()
+
+    async def run(self) -> None:
+        # Execute left pipeline
+        assert self._left is not None
+        await self._left.initialize()
+        await self._left.run()
+        await self._left.finish()
+        left_last = self._last_in_chain(self._left)
+        left_results: List[Dict[str, Any]] = left_last.results
+
+        # Execute right pipeline
+        assert self._right is not None
+        await self._right.initialize()
+        await self._right.run()
+        await self._right.finish()
+        right_last = self._last_in_chain(self._right)
+        right_results: List[Dict[str, Any]] = right_last.results
+
+        # Validate column names match
+        if left_results and right_results:
+            left_keys = sorted(left_results[0].keys())
+            right_keys = sorted(right_results[0].keys())
+            if left_keys != right_keys:
+                raise ValueError(
+                    "All sub queries in a UNION must have the same return column names"
+                )
+
+        # Combine results
+        self._results = self._combine(left_results, right_results)
+
+    def _combine(
+        self,
+        left: List[Dict[str, Any]],
+        right: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
+        """Combines results from left and right pipelines.
+
+        UNION removes duplicates; subclass UnionAll overrides to keep all rows.
+        """
+        combined = list(left)
+        for row in right:
+            serialized = json.dumps(row, sort_keys=True, default=str)
+            is_duplicate = any(
+                json.dumps(existing, sort_keys=True, default=str) == serialized
+                for existing in combined
+            )
+            if not is_duplicate:
+                combined.append(row)
+        return combined
+
+    async def finish(self) -> None:
+        if self.next:
+            await self.next.finish()
+
+    @property
+    def results(self) -> List[Dict[str, Any]]:
+        return self._results

package/flowquery-py/src/parsing/operations/union_all.py

@@ -0,0 +1,17 @@
+"""Represents a UNION ALL operation that concatenates results without deduplication."""
+
+from typing import Any, Dict, List
+
+from .union import Union
+
+
+class UnionAll(Union):
+    """Represents a UNION ALL operation that concatenates results from two sub-queries
+    without removing duplicates."""
+
+    def _combine(
+        self,
+        left: List[Dict[str, Any]],
+        right: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
+        return list(left) + list(right)

package/flowquery-py/src/parsing/parser.py

@@ -20,7 +20,6 @@ from .components.from_ import From
 from .components.headers import Headers
 from .components.null import Null
 from .components.post import Post
-from .context import Context
 from .data_structures.associative_array import AssociativeArray
 from .data_structures.json_array import JSONArray
 from .data_structures.key_value_pair import KeyValuePair
@@ -63,9 +62,12 @@ from .operations.load import Load
 from .operations.match import Match
 from .operations.operation import Operation
 from .operations.return_op import Return
+from .operations.union import Union
+from .operations.union_all import UnionAll
 from .operations.unwind import Unwind
 from .operations.where import Where
 from .operations.with_op import With
+from .parser_state import ParserState
 
 
 class Parser(BaseParser):
@@ -82,9 +84,7 @@ class Parser(BaseParser):
 
     def __init__(self, tokens: Optional[List[Token]] = None):
         super().__init__(tokens)
-        self._variables: Dict[str, ASTNode] = {}
-        self._context = Context()
-        self._returns = 0
+        self._state = ParserState()
 
     def parse(self, statement: str) -> ASTNode:
         """Parses a FlowQuery statement into an Abstract Syntax Tree.
@@ -112,13 +112,17 @@ class Parser(BaseParser):
             else:
                 self._skip_whitespace_and_comments()
 
+            # UNION separates two query pipelines — break and handle after the loop
+            if self.token.is_union():
+                break
+
             operation = self._parse_operation()
             if operation is None and not is_sub_query:
                 raise ValueError("Expected one of WITH, UNWIND, RETURN, LOAD, OR CALL")
             elif operation is None and is_sub_query:
                 return root
 
-            if self._returns > 1:
+            if self._state.returns > 1:
                 raise ValueError("Only one RETURN statement is allowed")
 
             if isinstance(previous, Call) and not previous.has_yield:
@@ -146,6 +150,26 @@ class Parser(BaseParser):
 
             previous = operation
 
+        # Handle UNION: wrap left and right pipelines into a Union node
+        if not self.token.is_eof() and self.token.is_union():
+            if not isinstance(operation, (Return, Call)):
+                raise ValueError(
+                    "Each side of UNION must end with a RETURN or CALL statement"
+                )
+            union = self._parse_union()
+            assert union is not None
+            union.left = root.first_child()  # type: ignore[assignment]
+            # Save and reset parser state for right-side scope
+            state: ParserState = self._state
+            self._state = ParserState()
+            right_root = self._parse_tokenized(is_sub_query)
+            union.right = right_root.first_child()  # type: ignore[assignment]
+            # Restore parser state
+            self._state = state
+            new_root = ASTNode()
+            new_root.add_child(union)
+            return new_root
+
         if not isinstance(operation, (Return, Call, CreateNode, CreateRelationship)):
             raise ValueError("Last statement must be a RETURN, WHERE, CALL, or CREATE statement")
 
@@ -199,7 +223,7 @@ class Parser(BaseParser):
         else:
             raise ValueError("Expected alias")
         unwind = Unwind(expression)
-        self._variables[alias.get_alias()] = unwind
+        self._state.variables[alias.get_alias()] = unwind
         return unwind
 
     def _parse_return(self) -> Optional[Return]:
@@ -217,7 +241,7 @@ class Parser(BaseParser):
             raise ValueError("Expected expression")
         if distinct or any(expr.has_reducers() for expr in expressions):
             return AggregatedReturn(expressions)
-        self._returns += 1
+        self._state.increment_returns()
         return Return(expressions)
 
     def _parse_where(self) -> Optional[Where]:
@@ -291,7 +315,7 @@ class Parser(BaseParser):
         alias = self._parse_alias()
         if alias is not None:
             load.add_child(alias)
-            self._variables[alias.get_alias()] = load
+            self._state.variables[alias.get_alias()] = load
         else:
             raise ValueError("Expected alias")
         return load
@@ -318,14 +342,21 @@ class Parser(BaseParser):
         return call
 
     def _parse_match(self) -> Optional[Match]:
+        optional = False
+        if self.token.is_optional():
+            optional = True
+            self.set_next_token()
+            self._expect_and_skip_whitespace_and_comments()
         if not self.token.is_match():
+            if optional:
+                raise ValueError("Expected MATCH after OPTIONAL")
             return None
         self.set_next_token()
         self._expect_and_skip_whitespace_and_comments()
         patterns = list(self._parse_patterns())
         if len(patterns) == 0:
             raise ValueError("Expected graph pattern")
-        return Match(patterns)
+        return Match(patterns, optional)
 
     def _parse_create(self) -> Optional[Operation]:
         """Parse CREATE VIRTUAL statement for nodes and relationships."""
@@ -383,6 +414,19 @@ class Parser(BaseParser):
         else:
             return CreateNode(node, query)
 
+    def _parse_union(self) -> Optional[Union]:
+        """Parse a UNION or UNION ALL keyword."""
+        if not self.token.is_union():
+            return None
+        self.set_next_token()
+        self._skip_whitespace_and_comments()
+        if self.token.is_all():
+            union: Union = UnionAll()
+            self.set_next_token()
+        else:
+            union = Union()
+        return union
+
     def _parse_sub_query(self) -> Optional[ASTNode]:
         """Parse a sub-query enclosed in braces."""
         if not self.token.is_opening_brace():
@@ -411,7 +455,7 @@ class Parser(BaseParser):
             if pattern is not None:
                 if identifier is not None:
                     pattern.identifier = identifier
-                    self._variables[identifier] = pattern
+                    self._state.variables[identifier] = pattern
                 yield pattern
             else:
                 break
@@ -491,8 +535,8 @@ class Parser(BaseParser):
         node = Node()
         node.label = label
         node.properties = dict(self._parse_properties())
-        if identifier is not None and identifier in self._variables:
-            reference = self._variables.get(identifier)
+        if identifier is not None and identifier in self._state.variables:
+            reference = self._state.variables.get(identifier)
             # Resolve through Expression -> Reference -> Node (e.g., after WITH)
             ref_child = reference.first_child() if isinstance(reference, Expression) else None
             if isinstance(ref_child, Reference):
@@ -504,7 +548,7 @@ class Parser(BaseParser):
             node = NodeReference(node, reference)
         elif identifier is not None:
             node.identifier = identifier
-            self._variables[identifier] = node
+            self._state.variables[identifier] = node
         if not self.token.is_right_parenthesis():
             raise ValueError("Expected closing parenthesis for node definition")
         self.set_next_token()
@@ -547,8 +591,8 @@ class Parser(BaseParser):
         relationship = Relationship()
         relationship.direction = direction
         relationship.properties = properties
-        if variable is not None and variable in self._variables:
-            reference = self._variables.get(variable)
+        if variable is not None and variable in self._state.variables:
+            reference = self._state.variables.get(variable)
             # Resolve through Expression -> Reference -> Relationship (e.g., after WITH)
             first = reference.first_child() if isinstance(reference, Expression) else None
             if isinstance(first, Reference):
@@ -560,7 +604,7 @@ class Parser(BaseParser):
             relationship = RelationshipReference(relationship, reference)
         elif variable is not None:
             relationship.identifier = variable
-            self._variables[variable] = relationship
+            self._state.variables[variable] = relationship
         if hops is not None:
             relationship.hops = hops
         relationship.type = rel_type
@@ -647,7 +691,7 @@ class Parser(BaseParser):
                     reference = expression.first_child()
                     assert isinstance(reference, Reference)  # For type narrowing
                     expression.set_alias(reference.identifier)
-                    self._variables[reference.identifier] = expression
+                    self._state.variables[reference.identifier] = expression
                 elif (alias_option == AliasOption.REQUIRED and
                       alias is None and
                       not isinstance(expression.first_child(), Reference)):
@@ -656,7 +700,7 @@ class Parser(BaseParser):
                     raise ValueError("Alias not allowed")
                 elif alias_option in (AliasOption.OPTIONAL, AliasOption.REQUIRED) and alias is not None:
                     expression.set_alias(alias.get_alias())
-                    self._variables[alias.get_alias()] = expression
+                    self._state.variables[alias.get_alias()] = expression
                 yield expression
             else:
                 break
@@ -670,7 +714,7 @@ class Parser(BaseParser):
         self._skip_whitespace_and_comments()
         if self.token.is_identifier_or_keyword() and (self.peek() is None or not self.peek().is_left_parenthesis()):
             identifier = self.token.value or ""
-            reference = Reference(identifier, self._variables.get(identifier))
+            reference = Reference(identifier, self._state.variables.get(identifier))
             self.set_next_token()
             lookup = self._parse_lookup(reference)
             expression.add_node(lookup)
@@ -1018,7 +1062,7 @@ class Parser(BaseParser):
         if not self.token.is_identifier():
             raise ValueError("Expected identifier")
         reference = Reference(self.token.value)
-        self._variables[reference.identifier] = reference
+        self._state.variables[reference.identifier] = reference
         func.add_child(reference)
         self.set_next_token()
         self._expect_and_skip_whitespace_and_comments()
@@ -1052,7 +1096,7 @@ class Parser(BaseParser):
         if not self.token.is_right_parenthesis():
             raise ValueError("Expected right parenthesis")
         self.set_next_token()
-        del self._variables[reference.identifier]
+        del self._state.variables[reference.identifier]
         return func
 
     def _parse_function(self) -> Optional[Function]:
@@ -1068,10 +1112,10 @@ class Parser(BaseParser):
             raise ValueError(f"Unknown function: {name}")
 
         # Check for nested aggregate functions
-        if isinstance(func, AggregateFunction) and self._context.contains_type(AggregateFunction):
+        if isinstance(func, AggregateFunction) and self._state.context.contains_type(AggregateFunction):
             raise ValueError("Aggregate functions cannot be nested")
 
-        self._context.push(func)
+        self._state.context.push(func)
         self.set_next_token()  # skip function name
         self.set_next_token()  # skip left parenthesis
         self._skip_whitespace_and_comments()
@@ -1088,7 +1132,7 @@ class Parser(BaseParser):
         if not self.token.is_right_parenthesis():
             raise ValueError("Expected right parenthesis")
         self.set_next_token()
-        self._context.pop()
+        self._state.context.pop()
         return func
 
     def _parse_async_function(self) -> Optional[AsyncFunction]:

package/flowquery-py/src/parsing/parser_state.py

@@ -0,0 +1,26 @@
+from typing import Dict
+
+from .ast_node import ASTNode
+from .context import Context
+
+
+class ParserState:
+    def __init__(self) -> None:
+        self._variables: Dict[str, ASTNode] = {}
+        self._context = Context()
+        self._returns = 0
+
+    @property
+    def variables(self) -> Dict[str, ASTNode]:
+        return self._variables
+
+    @property
+    def context(self) -> Context:
+        return self._context
+
+    @property
+    def returns(self) -> int:
+        return self._returns
+
+    def increment_returns(self) -> None:
+        self._returns += 1

package/flowquery-py/src/tokenization/keyword.py

@@ -8,6 +8,7 @@ class Keyword(Enum):
 
     RETURN = "RETURN"
     MATCH = "MATCH"
+    OPTIONAL = "OPTIONAL"
     WHERE = "WHERE"
     CREATE = "CREATE"
     VIRTUAL = "VIRTUAL"
@@ -49,3 +50,5 @@ class Keyword(Enum):
     CONTAINS = "CONTAINS"
     STARTS = "STARTS"
     ENDS = "ENDS"
+    UNION = "UNION"
+    ALL = "ALL"

package/flowquery-py/src/tokenization/token.py

@@ -462,6 +462,13 @@ class Token:
     def is_match(self) -> bool:
         return self._type == TokenType.KEYWORD and self._value == Keyword.MATCH.value
 
+    @staticmethod
+    def OPTIONAL() -> Token:
+        return Token(TokenType.KEYWORD, Keyword.OPTIONAL.value)
+
+    def is_optional(self) -> bool:
+        return self._type == TokenType.KEYWORD and self._value == Keyword.OPTIONAL.value
+
     @staticmethod
     def AS() -> Token:
         return Token(TokenType.KEYWORD, Keyword.AS.value)
@@ -609,6 +616,20 @@ class Token:
     def is_limit(self) -> bool:
         return self._type == TokenType.KEYWORD and self._value == Keyword.LIMIT.value
 
+    @staticmethod
+    def UNION() -> Token:
+        return Token(TokenType.KEYWORD, Keyword.UNION.value)
+
+    def is_union(self) -> bool:
+        return self._type == TokenType.KEYWORD and self._value == Keyword.UNION.value
+
+    @staticmethod
+    def ALL() -> Token:
+        return Token(TokenType.KEYWORD, Keyword.ALL.value)
+
+    def is_all(self) -> bool:
+        return self._type == TokenType.KEYWORD and self._value == Keyword.ALL.value
+
     # End of file token
 
     @staticmethod