flowquery 1.0.18 → 1.0.21

package/flowquery-py/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowquery"
-version = "1.0.7"
+version = "1.0.11"
 description = "A declarative query language for data processing pipelines"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -38,6 +38,11 @@ Issues = "https://github.com/microsoft/FlowQuery/issues"
 dev = [
     "pytest>=7.0.0",
     "pytest-asyncio>=0.21.0",
+    "jupyter>=1.0.0",
+    "ipykernel>=6.0.0",
+    "nbstripout>=0.6.0",
+    "mypy>=1.0.0",
+    "ruff>=0.1.0",
 ]
 
 [build-system]
@@ -72,4 +77,45 @@ python_functions = ["test_*"]
 addopts = "-v --tb=short"
 
 [tool.pytest-asyncio]
-mode = "auto"
+mode = "auto"
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+ignore_missing_imports = true
+exclude = [
+    "tests/",
+    "__pycache__",
+    ".git",
+    "build",
+    "dist",
+]
+
+[[tool.mypy.overrides]]
+module = "src.parsing.parser"
+warn_return_any = false
+disable_error_code = ["union-attr", "arg-type"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+exclude = [
+    ".git",
+    "__pycache__",
+    "build",
+    "dist",
+]
+
+[tool.ruff.lint]
+select = [
+    "F",      # Pyflakes (includes F401 unused imports)
+    "E",      # pycodestyle errors
+    "W",      # pycodestyle warnings
+    "I",      # isort
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["flowquery", "src"]

package/flowquery-py/src/__init__.py

@@ -2,21 +2,23 @@
 FlowQuery - A declarative query language for data processing pipelines.
 
 This is the Python implementation of FlowQuery.
+
+This module provides the core components for defining, parsing, and executing FlowQuery queries.
 """
 
 from .compute.runner import Runner
 from .io.command_line import CommandLine
-from .parsing.parser import Parser
-from .parsing.functions.function import Function
 from .parsing.functions.aggregate_function import AggregateFunction
 from .parsing.functions.async_function import AsyncFunction
-from .parsing.functions.predicate_function import PredicateFunction
-from .parsing.functions.reducer_element import ReducerElement
+from .parsing.functions.function import Function
 from .parsing.functions.function_metadata import (
+    FunctionCategory,
     FunctionDef,
     FunctionMetadata,
-    FunctionCategory,
 )
+from .parsing.functions.predicate_function import PredicateFunction
+from .parsing.functions.reducer_element import ReducerElement
+from .parsing.parser import Parser
 
 __all__ = [
     "Runner",

package/flowquery-py/src/compute/runner.py

@@ -9,10 +9,10 @@ from ..parsing.parser import Parser
 
 class Runner:
     """Executes a FlowQuery statement and retrieves the results.
-    
+
     The Runner class parses a FlowQuery statement into an AST and executes it,
     managing the execution flow from the first operation to the final return statement.
-    
+
     Example:
         runner = Runner("WITH 1 as x RETURN x")
         await runner.run()
@@ -25,24 +25,28 @@ class Runner:
         ast: Optional[ASTNode] = None
     ):
         """Creates a new Runner instance and parses the FlowQuery statement.
-        
+
         Args:
             statement: The FlowQuery statement to execute
             ast: An already-parsed AST (optional)
-            
+
         Raises:
             ValueError: If neither statement nor AST is provided
         """
         if (statement is None or statement == "") and ast is None:
             raise ValueError("Either statement or AST must be provided")
-        
-        _ast = ast if ast is not None else Parser().parse(statement)
-        self._first: Operation = _ast.first_child()
-        self._last: Operation = _ast.last_child()
+
+        _ast = ast if ast is not None else Parser().parse(statement or "")
+        first = _ast.first_child()
+        last = _ast.last_child()
+        if not isinstance(first, Operation) or not isinstance(last, Operation):
+            raise ValueError("AST must contain Operations")
+        self._first: Operation = first
+        self._last: Operation = last
 
     async def run(self) -> None:
         """Executes the parsed FlowQuery statement.
-        
+
         Raises:
             Exception: If an error occurs during execution
         """
@@ -53,7 +57,7 @@ class Runner:
     @property
     def results(self) -> List[Dict[str, Any]]:
         """Gets the results from the executed statement.
-        
+
         Returns:
             The results from the last operation (typically a RETURN statement)
         """

package/flowquery-py/src/extensibility.py

@@ -4,7 +4,7 @@ This module provides all the exports needed to create custom FlowQuery functions
 
 Example:
     from flowquery.extensibility import Function, FunctionDef
-    
+
     @FunctionDef({
         'description': "Converts a string to uppercase",
         'category': "string",
@@ -15,27 +15,27 @@ Example:
         def __init__(self):
             super().__init__("uppercase")
             self._expected_parameter_count = 1
-        
+
         def value(self) -> str:
             return str(self.get_children()[0].value()).upper()
 """
 
 # Base function classes for creating custom functions
-from .parsing.functions.function import Function
 from .parsing.functions.aggregate_function import AggregateFunction
 from .parsing.functions.async_function import AsyncFunction
-from .parsing.functions.predicate_function import PredicateFunction
-from .parsing.functions.reducer_element import ReducerElement
+from .parsing.functions.function import Function
 
 # Decorator and metadata types for function registration
 from .parsing.functions.function_metadata import (
+    FunctionCategory,
     FunctionDef,
-    FunctionMetadata,
     FunctionDefOptions,
-    ParameterSchema,
+    FunctionMetadata,
     OutputSchema,
-    FunctionCategory,
+    ParameterSchema,
 )
+from .parsing.functions.predicate_function import PredicateFunction
+from .parsing.functions.reducer_element import ReducerElement
 
 __all__ = [
     "Function",

package/flowquery-py/src/graph/__init__.py

@@ -1,18 +1,18 @@
 """Graph module for FlowQuery."""
 
-from .node import Node
-from .relationship import Relationship
-from .pattern import Pattern
-from .patterns import Patterns
-from .pattern_expression import PatternExpression
 from .database import Database
 from .hops import Hops
+from .node import Node
 from .node_data import NodeData
 from .node_reference import NodeReference
-from .relationship_data import RelationshipData
-from .relationship_reference import RelationshipReference
+from .pattern import Pattern
+from .pattern_expression import PatternExpression
+from .patterns import Patterns
 from .physical_node import PhysicalNode
 from .physical_relationship import PhysicalRelationship
+from .relationship import Relationship
+from .relationship_data import RelationshipData
+from .relationship_reference import RelationshipReference
 
 __all__ = [
     "Node",

package/flowquery-py/src/graph/data.py

@@ -38,14 +38,20 @@ class IndexEntry:
 class Layer:
     """Layer for managing index state at a specific level."""
 
-    def __init__(self, index: Dict[str, IndexEntry]):
-        self._index: Dict[str, IndexEntry] = index
+    def __init__(self, indexes: Dict[str, Dict[str, IndexEntry]]):
+        self._indexes: Dict[str, Dict[str, IndexEntry]] = indexes
         self._current: int = -1
 
+    def index(self, name: str) -> Dict[str, IndexEntry]:
+        """Get or create an index by name."""
+        if name not in self._indexes:
+            self._indexes[name] = {}
+        return self._indexes[name]
+
     @property
-    def index(self) -> Dict[str, IndexEntry]:
-        """Get the index dictionary."""
-        return self._index
+    def indexes(self) -> Dict[str, Dict[str, IndexEntry]]:
+        """Get all indexes."""
+        return self._indexes
 
     @property
     def current(self) -> int:
@@ -67,30 +73,40 @@ class Data:
 
     def _build_index(self, key: str, level: int = 0) -> None:
         """Build an index for the given key at the specified level."""
-        self.layer(level).index.clear()
-        for idx, record in enumerate(self._records):
+        idx = self.layer(level).index(key)
+        idx.clear()
+        for i, record in enumerate(self._records):
             if key in record:
-                if record[key] not in self.layer(level).index:
-                    self.layer(level).index[record[key]] = IndexEntry()
-                self.layer(level).index[record[key]].add(idx)
+                if record[key] not in idx:
+                    idx[record[key]] = IndexEntry()
+                idx[record[key]].add(i)
 
     def layer(self, level: int = 0) -> Layer:
         """Get or create a layer at the specified level."""
         if level not in self._layers:
             first = self._layers[0]
-            cloned = {}
-            for key, entry in first.index.items():
-                cloned[key] = entry.clone()
-            self._layers[level] = Layer(cloned)
+            cloned_indexes = {}
+            for name, index_map in first.indexes.items():
+                cloned_map = {}
+                for key, entry in index_map.items():
+                    cloned_map[key] = entry.clone()
+                cloned_indexes[name] = cloned_map
+            self._layers[level] = Layer(cloned_indexes)
         return self._layers[level]
 
-    def _find(self, key: str, level: int = 0) -> bool:
+    def _find(self, key: str, level: int = 0, index_name: Optional[str] = None) -> bool:
         """Find the next record with the given key value."""
-        if key not in self.layer(level).index:
+        idx: Optional[Dict[str, IndexEntry]] = None
+        if index_name:
+            idx = self.layer(level).index(index_name)
+        else:
+            indexes = self.layer(level).indexes
+            idx = next(iter(indexes.values())) if indexes else None
+        if not idx or key not in idx:
             self.layer(level).current = len(self._records)  # Move to end
             return False
         else:
-            entry = self.layer(level).index[key]
+            entry = idx[key]
             more = entry.next()
             if not more:
                 self.layer(level).current = len(self._records)  # Move to end
@@ -100,9 +116,11 @@ class Data:
 
     def reset(self) -> None:
         """Reset iteration to the beginning."""
-        self.layer(0).current = -1
-        for entry in self.layer(0).index.values():
-            entry.reset()
+        for layer in self._layers.values():
+            layer.current = -1
+            for index_map in layer.indexes.values():
+                for entry in index_map.values():
+                    entry.reset()
 
     def next(self, level: int = 0) -> bool:
         """Move to the next record. Returns True if successful."""

package/flowquery-py/src/graph/database.py

@@ -1,14 +1,16 @@
 """Graph database for FlowQuery."""
 
-from typing import Any, Dict, Optional, Union, TYPE_CHECKING
+from __future__ import annotations
 
-from ..parsing.ast_node import ASTNode
+from typing import Dict, Optional, Union
 
-if TYPE_CHECKING:
-    from .node import Node
-    from .relationship import Relationship
-    from .node_data import NodeData
-    from .relationship_data import RelationshipData
+from ..parsing.ast_node import ASTNode
+from .node import Node
+from .node_data import NodeData
+from .physical_node import PhysicalNode
+from .physical_relationship import PhysicalRelationship
+from .relationship import Relationship
+from .relationship_data import RelationshipData
 
 
 class Database:
@@ -18,7 +20,7 @@ class Database:
     _nodes: Dict[str, 'PhysicalNode'] = {}
     _relationships: Dict[str, 'PhysicalRelationship'] = {}
 
-    def __init__(self):
+    def __init__(self) -> None:
         pass
 
     @classmethod
@@ -29,7 +31,6 @@ class Database:
 
     def add_node(self, node: 'Node', statement: ASTNode) -> None:
         """Adds a node to the database."""
-        from .physical_node import PhysicalNode
         if node.label is None:
             raise ValueError("Node label is null")
         physical = PhysicalNode(None, node.label)
@@ -42,7 +43,6 @@ class Database:
 
     def add_relationship(self, relationship: 'Relationship', statement: ASTNode) -> None:
         """Adds a relationship to the database."""
-        from .physical_relationship import PhysicalRelationship
         if relationship.type is None:
             raise ValueError("Relationship type is null")
         physical = PhysicalRelationship()
@@ -56,11 +56,6 @@ class Database:
 
     async def get_data(self, element: Union['Node', 'Relationship']) -> Union['NodeData', 'RelationshipData']:
         """Gets data for a node or relationship."""
-        from .node import Node
-        from .relationship import Relationship
-        from .node_data import NodeData
-        from .relationship_data import RelationshipData
-        
         if isinstance(element, Node):
             node = self.get_node(element)
             if node is None:
@@ -75,8 +70,3 @@ class Database:
             return RelationshipData(data)
         else:
             raise ValueError("Element is neither Node nor Relationship")
-
-
-# Import for type hints
-from .physical_node import PhysicalNode
-from .physical_relationship import PhysicalRelationship

package/flowquery-py/src/graph/node.py

@@ -1,13 +1,15 @@
 """Graph node representation for FlowQuery."""
 
-from typing import Any, Callable, Dict, Optional, TYPE_CHECKING
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Union
 
 from ..parsing.ast_node import ASTNode
 from ..parsing.expressions.expression import Expression
+from .node_data import NodeData, NodeRecord
 
 if TYPE_CHECKING:
     from .relationship import Relationship
-    from .node_data import NodeData, NodeRecord
 
 
 class Node(ASTNode):
@@ -26,7 +28,7 @@ class Node(ASTNode):
         self._incoming: Optional['Relationship'] = None
         self._outgoing: Optional['Relationship'] = None
         self._data: Optional['NodeData'] = None
-        self._todo_next: Optional[Callable[[], None]] = None
+        self._todo_next: Optional[Callable[[], Union[None, Awaitable[None]]]] = None
 
     @property
     def identifier(self) -> Optional[str]:
@@ -41,21 +43,40 @@ class Node(ASTNode):
         return self._label
 
     @label.setter
-    def label(self, value: str) -> None:
+    def label(self, value: Optional[str]) -> None:
         self._label = value
 
     @property
     def properties(self) -> Dict[str, Expression]:
         return self._properties
 
+    @properties.setter
+    def properties(self, value: Dict[str, Expression]) -> None:
+        self._properties = value
+
     def set_property(self, key: str, value: Expression) -> None:
         self._properties[key] = value
 
     def get_property(self, key: str) -> Optional[Expression]:
         return self._properties.get(key)
 
-    def set_value(self, value: 'NodeRecord') -> None:
-        self._value = value
+    def _matches_properties(self, hop: int = 0) -> bool:
+        """Check if current record matches all constraint properties."""
+        if not self._properties:
+            return True
+        if self._data is None:
+            return True
+        for key, expression in self._properties.items():
+            record = self._data.current(hop)
+            if record is None:
+                raise ValueError("No current node data available")
+            if key not in record:
+                raise ValueError("Node does not have property")
+            return bool(record[key] == expression.value())
+        return True
+
+    def set_value(self, value: Dict[str, Any]) -> None:
+        self._value = value  # type: ignore[assignment]
 
     def value(self) -> Optional['NodeRecord']:
         return self._value
@@ -83,30 +104,40 @@ class Node(ASTNode):
         if self._data:
             self._data.reset()
             while self._data.next():
-                self.set_value(self._data.current())
-                if self._outgoing:
-                    await self._outgoing.find(self._value['id'])
-                await self.run_todo_next()
+                current = self._data.current()
+                if current is not None:
+                    self.set_value(current)
+                    if not self._matches_properties():
+                        continue
+                    if self._outgoing and self._value:
+                        await self._outgoing.find(self._value['id'])
+                    await self.run_todo_next()
 
     async def find(self, id_: str, hop: int = 0) -> None:
         if self._data:
             self._data.reset()
             while self._data.find(id_, hop):
-                self.set_value(self._data.current(hop))
-                if self._incoming:
-                    self._incoming.set_end_node(self)
-                if self._outgoing:
-                    await self._outgoing.find(self._value['id'], hop)
-                await self.run_todo_next()
+                current = self._data.current(hop)
+                if current is not None:
+                    self.set_value(current)
+                    if not self._matches_properties(hop):
+                        continue
+                    if self._incoming:
+                        self._incoming.set_end_node(self)
+                    if self._outgoing and self._value:
+                        await self._outgoing.find(self._value['id'], hop)
+                    await self.run_todo_next()
 
     @property
-    def todo_next(self) -> Optional[Callable[[], None]]:
+    def todo_next(self) -> Optional[Callable[[], Union[None, Awaitable[None]]]]:
         return self._todo_next
 
     @todo_next.setter
-    def todo_next(self, func: Optional[Callable[[], None]]) -> None:
+    def todo_next(self, func: Optional[Callable[[], Union[None, Awaitable[None]]]]) -> None:
         self._todo_next = func
 
     async def run_todo_next(self) -> None:
         if self._todo_next:
-            await self._todo_next()
+            result = self._todo_next()
+            if result is not None:
+                await result

package/flowquery-py/src/graph/node_data.py

@@ -19,7 +19,7 @@ class NodeData(Data):
 
     def find(self, id_: str, hop: int = 0) -> bool:
         """Find a record by ID."""
-        return self._find(id_, hop)
+        return self._find(id_, hop, "id")
 
     def current(self, hop: int = 0) -> Optional[Dict[str, Any]]:
         """Get the current record."""

package/flowquery-py/src/graph/node_reference.py

@@ -1,17 +1,14 @@
-"""Node reference for FlowQuery."""
+from __future__ import annotations
 
-from typing import Optional, TYPE_CHECKING
+from typing import Any, Optional
 
 from .node import Node
 
-if TYPE_CHECKING:
-    from ..parsing.ast_node import ASTNode
-
 
 class NodeReference(Node):
     """Represents a reference to an existing node variable."""
 
-    def __init__(self, base: Node, reference: Node):
+    def __init__(self, base: Node, reference: Node) -> None:
         super().__init__(base.identifier, base.label)
         self._reference: Node = reference
         # Copy properties from base
@@ -28,14 +25,16 @@ class NodeReference(Node):
     def referred(self) -> Node:
         return self._reference
 
-    def value(self):
+    def value(self) -> Optional[Any]:
         return self._reference.value() if self._reference else None
 
     async def next(self) -> None:
         """Process next using the referenced node's value."""
-        self.set_value(self._reference.value())
-        if self._outgoing and self._value:
-            await self._outgoing.find(self._value['id'])
+        ref_value = self._reference.value()
+        if ref_value is not None:
+            self.set_value(dict(ref_value))
+            if self._outgoing and self._value:
+                await self._outgoing.find(self._value['id'])
         await self.run_todo_next()
 
     async def find(self, id_: str, hop: int = 0) -> None:
@@ -43,7 +42,7 @@ class NodeReference(Node):
         referenced = self._reference.value()
         if referenced is None or id_ != referenced.get('id'):
             return
-        self.set_value(referenced)
+        self.set_value(dict(referenced))
         if self._outgoing and self._value:
             await self._outgoing.find(self._value['id'], hop)
         await self.run_todo_next()