flowquery 1.0.17 → 1.0.20

package/flowquery-py/notebooks/TestFlowQuery.ipynb

@@ -0,0 +1,440 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "from flowquery import Runner"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1",
+   "metadata": {},
+   "source": [
+    "Create virtual node test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query: str = \"\"\"\n",
+    "    create virtual (:Fact) as {\n",
+    "        unwind range(0,10) as i\n",
+    "        load json from \"https://catfact.ninja/fact\" as item\n",
+    "        return i as id, item.fact\n",
+    "    }\n",
+    "\"\"\"\n",
+    "\n",
+    "runner: Runner = Runner(query)\n",
+    "await runner.run()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3",
+   "metadata": {},
+   "source": [
+    "Query virtual graph node"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runner: Runner = Runner(\"match (n:Fact) return n\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5",
+   "metadata": {},
+   "source": [
+    "Test extensibility"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from flowquery.extensibility import (\n",
+    "    Function,\n",
+    "    FunctionDef,\n",
+    "    AggregateFunction,\n",
+    "    ReducerElement,\n",
+    "    AsyncFunction,\n",
+    "    PredicateFunction\n",
+    ")\n",
+    "import aiohttp\n",
+    "import json\n",
+    "from typing import Any, List, Iterator, Union, Dict\n",
+    "\n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Converts a string to uppercase\",\n",
+    "    \"category\": \"string\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"text\", \"description\": \"String to convert\", \"type\": \"string\"}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"Uppercase string\", \"type\": \"string\"}\n",
+    "})\n",
+    "class UpperCase(Function):\n",
+    "    def __init__(self):\n",
+    "        super().__init__(\"uppercase\")\n",
+    "        self._expected_parameter_count = 1\n",
+    "\n",
+    "    def value(self) -> str:\n",
+    "        return str(self.get_children()[0].value()).upper()\n",
+    "    \n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Extracts nodes from a collection\",\n",
+    "    \"category\": \"scalar\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"collection\", \"description\": \"Collection to extract nodes from\", \"type\": \"any[]\"}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"List of nodes extracted from the collection\", \"type\": \"node[]\"}\n",
+    "})\n",
+    "class Nodes(Function):\n",
+    "    def __init__(self):\n",
+    "        super().__init__(\"nodes\")\n",
+    "        self._expected_parameter_count = 1\n",
+    "\n",
+    "    def value(self) -> List[Dict[str, Any]]:\n",
+    "        pattern: List[Dict[str, Any]] = self.get_children()[0].value()\n",
+    "        return list(self._nodes(pattern))\n",
+    "\n",
+    "    def _nodes(self, pattern: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]:\n",
+    "        for element in pattern:\n",
+    "            if isinstance(element, dict) and \"id\" in element:\n",
+    "                yield element\n",
+    "    \n",
+    "\n",
+    "class ProductElement(ReducerElement):\n",
+    "    def __init__(self):\n",
+    "        self._value: float = 1.0\n",
+    "\n",
+    "    @property\n",
+    "    def value(self) -> float:\n",
+    "        return self._value\n",
+    "\n",
+    "    @value.setter\n",
+    "    def value(self, v: float) -> None:\n",
+    "        self._value = v\n",
+    "    \n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Calculates the product of a list of numbers\",\n",
+    "    \"category\": \"aggregate\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"numbers\", \"description\": \"List of numbers to multiply\", \"type\": \"number[]\"}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"Product of the numbers\", \"type\": \"number\"}\n",
+    "})\n",
+    "class Product(AggregateFunction):\n",
+    "    def __init__(self):\n",
+    "        super().__init__(\"product\")\n",
+    "\n",
+    "    def reduce(self, element: ReducerElement) -> None:\n",
+    "        element.value *= self.first_child().value()\n",
+    "\n",
+    "    def element(self) -> ReducerElement:\n",
+    "        return ProductElement()\n",
+    "\n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Asynchronous function that fetches data from a URL\",\n",
+    "    \"category\": \"async\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"url\", \"description\": \"URL to fetch data from\", \"type\": \"string\"}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"Fetched data\", \"type\": \"string\"}\n",
+    "})\n",
+    "class get(AsyncFunction):\n",
+    "    async def generate(self, url: str):\n",
+    "        async with aiohttp.ClientSession() as session:\n",
+    "            async with session.get(url) as response:\n",
+    "                yield await response.json()\n",
+    "\n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Fetch json data from a file path\",\n",
+    "    \"category\": \"async\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"path\", \"description\": \"File path to fetch data from\", \"type\": \"string\"}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"Fetched data\", \"type\": \"string\"}\n",
+    "})\n",
+    "class json_file(AsyncFunction):\n",
+    "    async def generate(self, path: str):\n",
+    "        with open(path, \"r\") as file:\n",
+    "            yield json.load(file)\n",
+    "\n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Extracts values from an array with optional filtering. Uses list comprehension syntax: extract(variable IN array [WHERE condition] | expression)\",\n",
+    "    \"category\": \"predicate\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"variable\", \"description\": \"Variable name to bind each element\", \"type\": \"string\"},\n",
+    "        {\"name\": \"array\", \"description\": \"Array to iterate over\", \"type\": \"array\"},\n",
+    "        {\"name\": \"expression\", \"description\": \"Expression to return for each element\", \"type\": \"any\"},\n",
+    "        {\"name\": \"where\", \"description\": \"Optional filter condition\", \"type\": \"boolean\", \"required\": False}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"Extracted values from the array after applying the optional filter\", \"type\": \"array\", \"example\": [2, 4]},\n",
+    "    \"examples\": [\n",
+    "        \"WITH [1, 2, 3] AS nums RETURN extract(n IN nums | n)\",\n",
+    "        \"WITH [1, 2, 3, 4] AS nums RETURN extract(n IN nums WHERE n > 1 | n * 2)\"\n",
+    "    ]\n",
+    "})\n",
+    "class PredicateExtract(PredicateFunction):\n",
+    "    \"\"\"PredicateExtract function.\n",
+    "    \n",
+    "    Extracts values from an array with optional filtering.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self):\n",
+    "        super().__init__(\"extract\")\n",
+    "\n",
+    "    def value(self) -> List[Any]:\n",
+    "        return list(self._extract())\n",
+    "    \n",
+    "    def _extract(self) -> Iterator[Any]:\n",
+    "        self.reference.referred = self._value_holder\n",
+    "        array = self.array.value()\n",
+    "        if array is None or not isinstance(array, list):\n",
+    "            raise ValueError(\"Invalid array for extract function\")\n",
+    "        \n",
+    "        for item in array:\n",
+    "            self._value_holder.holder = item\n",
+    "            if self.where is None or self.where.value():\n",
+    "                yield self._return.value()\n",
+    "\n",
+    "@FunctionDef({\n",
+    "    \"description\": \"Checks if any element in the array satisfies the condition. Uses list comprehension syntax: any(variable IN array [WHERE condition])\",\n",
+    "    \"category\": \"predicate\",\n",
+    "    \"parameters\": [\n",
+    "        {\"name\": \"variable\", \"description\": \"Variable name to bind each element\", \"type\": \"string\"},\n",
+    "        {\"name\": \"array\", \"description\": \"Array to iterate over\", \"type\": \"array\"},\n",
+    "        {\"name\": \"where\", \"description\": \"Condition to check for each element\", \"type\": \"boolean\", \"required\": False}\n",
+    "    ],\n",
+    "    \"output\": {\"description\": \"True if any element satisfies the condition, otherwise false\", \"type\": \"boolean\", \"example\": True},\n",
+    "    \"examples\": [\n",
+    "        \"WITH [1, 2, 3] AS nums RETURN any(n IN nums | n > 2)\",\n",
+    "        \"WITH [1, 2, 3] AS nums RETURN any(n IN nums | n > 5)\"\n",
+    "    ]\n",
+    "})\n",
+    "class Any(PredicateFunction):\n",
+    "    \"\"\"Any function.\n",
+    "    \n",
+    "    Returns true if any element in the array satisfies the condition.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self):\n",
+    "        super().__init__(\"any\")\n",
+    "\n",
+    "    def value(self) -> bool:\n",
+    "        return any(self._any())\n",
+    "    \n",
+    "    def _any(self) -> Iterator[bool]:\n",
+    "        self.reference.referred = self._value_holder\n",
+    "        array = self.array.value()\n",
+    "        if array is None or not isinstance(array, list):\n",
+    "            raise ValueError(\"Invalid array for any function\")\n",
+    "        \n",
+    "        for item in array:\n",
+    "            self._value_holder.holder = item\n",
+    "            if self.where is None or self.where.value():\n",
+    "                yield True"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7",
+   "metadata": {},
+   "source": [
+    "Test functions just created"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runner: Runner = Runner(\"\"\"\n",
+    "    return uppercase(\"hello world\") as uppercased\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    unwind [1, 2, 3, 4, 5] as num\n",
+    "    return product(num) as total_product\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    load json from get(\"https://catfact.ninja/fact\") as result\n",
+    "    return result.fact as cat_fact\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    load json from json_file(\"../misc/data/test.json\") as result\n",
+    "    unwind result as entry\n",
+    "    return entry\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    with [\n",
+    "        {\"age\": 25, \"name\": \"Alice\"},\n",
+    "        {\"age\": 30, \"name\": \"Bob\"},\n",
+    "        {\"age\": 22, \"name\": \"Charlie\"},\n",
+    "        {\"age\": 28, \"name\": \"Diana\"}\n",
+    "    ] as people\n",
+    "    return extract(p IN people | p.name WHERE p.age > 25) as names_over_25\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    with [1, 2, 3, 4, 5] as numbers\n",
+    "    return any(n IN numbers | n where n > 6) as has_greater_than_3\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "List functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runner: Runner = Runner(\"\"\"\n",
+    "    unwind functions() as func\n",
+    "    return func\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "for record in runner.results:\n",
+    "    print(record)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11",
+   "metadata": {},
+   "source": [
+    "Test virtual graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "await Runner(\"\"\"\n",
+    "    create virtual (:User) as {\n",
+    "        load json from json_file(\n",
+    "            \"../misc/data/users.json\"\n",
+    "        ) as users\n",
+    "        unwind users as user\n",
+    "        return user.id as id,\n",
+    "            user.name as name,\n",
+    "            user.title as title,\n",
+    "            user.department as department,\n",
+    "            user.email as email,\n",
+    "            user.managerId as managerId\n",
+    "    }\n",
+    "\"\"\").run()\n",
+    "\n",
+    "await Runner(\"\"\"\n",
+    "    create virtual (:User)-[:MANAGED_BY]->(:User) as {\n",
+    "        load json from json_file(\n",
+    "            \"../misc/data/users.json\"\n",
+    "        ) as users\n",
+    "        unwind users as user\n",
+    "        return user.id as left_id, user.managerId as right_id\n",
+    "    }\n",
+    "\"\"\").run()\n",
+    "\n",
+    "runner: Runner = Runner(\"\"\"\n",
+    "    MATCH p=(u:User)-[:MANAGED_BY*]->(ceo:User)\n",
+    "    WHERE NOT (ceo)-[:MANAGED_BY]->(:User)\n",
+    "    and any(n IN nodes(p) | n where n.department = \"Litigation\")\n",
+    "    RETURN\n",
+    "        u.name as employee,\n",
+    "        extract(n IN nodes(p) | n.name) as management_chain\n",
+    "\"\"\")\n",
+    "await runner.run()\n",
+    "print(f\"Total results: {len(runner.results)}\")\n",
+    "for record in runner.results:\n",
+    "    print(record)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "flowquery",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

package/flowquery-py/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowquery"
-version = "1.0.0"
+version = "1.0.9"
 description = "A declarative query language for data processing pipelines"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -38,6 +38,9 @@ Issues = "https://github.com/microsoft/FlowQuery/issues"
 dev = [
     "pytest>=7.0.0",
     "pytest-asyncio>=0.21.0",
+    "jupyter>=1.0.0",
+    "ipykernel>=6.0.0",
+    "nbstripout>=0.6.0",
 ]
 
 [build-system]

package/flowquery-py/src/__init__.py

@@ -2,6 +2,8 @@
 FlowQuery - A declarative query language for data processing pipelines.
 
 This is the Python implementation of FlowQuery.
+
+This module provides the core components for defining, parsing, and executing FlowQuery queries.
 """
 
 from .compute.runner import Runner

package/flowquery-py/src/graph/data.py

@@ -100,9 +100,10 @@ class Data:
 
     def reset(self) -> None:
         """Reset iteration to the beginning."""
-        self.layer(0).current = -1
-        for entry in self.layer(0).index.values():
-            entry.reset()
+        for layer in self._layers.values():
+            layer.current = -1
+            for entry in layer.index.values():
+                entry.reset()
 
     def next(self, level: int = 0) -> bool:
         """Move to the next record. Returns True if successful."""

package/flowquery-py/src/graph/pattern.py

@@ -85,16 +85,19 @@ class Pattern(ASTNode):
         from .node import Node
         from .relationship import Relationship
         
-        for element in self._chain:
+        for i, element in enumerate(self._chain):
             if isinstance(element, Node):
+                # Skip node if previous element was a zero-hop relationship (no matches)
+                if i > 0 and isinstance(self._chain[i-1], Relationship) and len(self._chain[i-1].matches) == 0:
+                    continue
                 yield element.value()
             elif isinstance(element, Relationship):
-                i = 0
+                j = 0
                 for match in element.matches:
                     yield match
-                    if i < len(element.matches) - 1:
+                    if j < len(element.matches) - 1:
                         yield match["endNode"]
-                    i += 1
+                    j += 1
 
     async def fetch_data(self) -> None:
         """Loads data from the database for all elements."""

package/flowquery-py/src/graph/pattern_expression.py

@@ -20,11 +20,14 @@ class PatternExpression(Pattern):
         self._evaluation: bool = False
 
     def add_element(self, element) -> None:
-        """Add an element to the pattern, ensuring it starts with a NodeReference."""
-        if len(self._chain) == 0 and not isinstance(element, NodeReference):
-            raise ValueError("PatternExpression must start with a NodeReference")
         super().add_element(element)
 
+    def verify(self) -> None:
+        if(len(self._chain) == 0):
+            raise ValueError("PatternExpression cannot be empty")
+        if not(any(isinstance(el, NodeReference) for el in self._chain if isinstance(el, ASTNode))):
+            raise ValueError("PatternExpression must contain at least one NodeReference")
+
     @property
     def identifier(self):
         return None

package/flowquery-py/src/graph/relationship.py

@@ -118,6 +118,13 @@ class Relationship(ASTNode):
             self._source = self._target
         if hop == 0:
             self._data.reset() if self._data else None
+            
+            # Handle zero-hop case: when min is 0 on a variable-length relationship,
+            # match source node as target (no traversal)
+            if self._hops and self._hops.multi() and self._hops.min == 0 and self._target:
+                # For zero-hop, target finds the same node as source (left_id)
+                # No relationship match is pushed since no edge is traversed
+                await self._target.find(left_id, hop)
         
         while self._data and self._data.find(left_id, hop):
             data = self._data.current(hop)

package/flowquery-py/src/io/command_line.py

@@ -1,5 +1,6 @@
 """Interactive command-line interface for FlowQuery."""
 
+import argparse
 import asyncio
 from typing import Optional
 
@@ -15,8 +16,26 @@ class CommandLine:
     Example:
         cli = CommandLine()
         cli.loop()  # Starts interactive mode
+        
+        # Or execute a single query:
+        cli.execute("load json from 'https://example.com/data' as d return d")
     """
 
+    def execute(self, query: str) -> None:
+        """Execute a single FlowQuery statement and print results.
+        
+        Args:
+            query: The FlowQuery statement to execute.
+        """
+        # Remove the termination semicolon if present
+        query = query.strip().rstrip(";")
+        
+        try:
+            runner = Runner(query)
+            asyncio.run(self._execute(runner))
+        except Exception as e:
+            print(f"Error: {e}")
+
     def loop(self) -> None:
         """Starts the interactive command loop.
         
@@ -63,5 +82,28 @@ class CommandLine:
 
 
 def main() -> None:
-    """Entry point for the flowquery CLI command."""
-    CommandLine().loop()
+    """Entry point for the flowquery CLI command.
+    
+    Usage:
+        flowquery              # Start interactive mode
+        flowquery -c "query"   # Execute a single query
+        flowquery --command "query"
+    """
+    parser = argparse.ArgumentParser(
+        description="FlowQuery - A declarative query language for data processing pipelines",
+        prog="flowquery"
+    )
+    parser.add_argument(
+        "-c", "--command",
+        type=str,
+        metavar="QUERY",
+        help="Execute a FlowQuery statement and exit"
+    )
+    
+    args = parser.parse_args()
+    cli = CommandLine()
+    
+    if args.command:
+        cli.execute(args.command)
+    else:
+        cli.loop()

package/flowquery-py/src/parsing/base_parser.py

@@ -69,7 +69,7 @@ class BaseParser:
             The current token, or EOF if at the end
         """
         if self._token_index >= len(self._tokens):
-            return Token.EOF
+            return Token.EOF()
         return self._tokens[self._token_index]
 
     @property
@@ -80,5 +80,5 @@ class BaseParser:
             The previous token, or EOF if at the beginning
         """
         if self._token_index - 1 < 0:
-            return Token.EOF
+            return Token.EOF()
         return self._tokens[self._token_index - 1]

package/flowquery-py/src/parsing/operations/load.py

@@ -96,6 +96,12 @@ class Load(Operation):
             headers = options.pop("headers", {})
             body = options.pop("body", None)
             
+            # Set Accept-Encoding to support common compression formats
+            # Note: brotli (br) is excluded due to API incompatibility between
+            # aiohttp 3.13+ and the brotli package's Decompressor.decompress() method
+            if "Accept-Encoding" not in headers:
+                headers["Accept-Encoding"] = "gzip, deflate"
+            
             async with session.request(
                 method, 
                 self.from_,