flowquery 1.0.37 → 1.0.39

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",

package/src/parsing/expressions/operator.ts

@@ -197,7 +197,7 @@ class Not extends Operator {
 
 class Is extends Operator {
     constructor() {
-        super(-1, true);
+        super(0, true);
     }
     public value(): number {
         return this.lhs.value() == this.rhs.value() ? 1 : 0;
@@ -206,7 +206,7 @@ class Is extends Operator {
 
 class IsNot extends Operator {
     constructor() {
-        super(-1, true);
+        super(0, true);
     }
     public value(): number {
         return this.lhs.value() != this.rhs.value() ? 1 : 0;
@@ -215,7 +215,7 @@ class IsNot extends Operator {
 
 class In extends Operator {
     constructor() {
-        super(-1, true);
+        super(0, true);
     }
     public value(): number {
         const list = this.rhs.value();
@@ -228,7 +228,7 @@ class In extends Operator {
 
 class NotIn extends Operator {
     constructor() {
-        super(-1, true);
+        super(0, true);
     }
     public value(): number {
         const list = this.rhs.value();

package/src/parsing/operations/aggregated_return.ts

@@ -1,6 +1,6 @@
-import Return from "./return";
-import GroupBy from "./group_by";
 import Expression from "../expressions/expression";
+import GroupBy from "./group_by";
+import Return from "./return";
 
 class AggregatedReturn extends Return {
     private _group_by: GroupBy = new GroupBy(this.children as Expression[]);
@@ -8,11 +8,15 @@ class AggregatedReturn extends Return {
         await this._group_by.run();
     }
     public get results(): Record<string, any>[] {
-        if(this._where !== null) {
+        if (this._where !== null) {
             this._group_by.where = this._where;
         }
-        return Array.from(this._group_by.generate_results());
+        const results = Array.from(this._group_by.generate_results());
+        if (this._orderBy !== null) {
+            return this._orderBy.sort(results);
+        }
+        return results;
     }
 }
 
-export default AggregatedReturn;
+export default AggregatedReturn;

package/src/parsing/operations/group_by.ts

@@ -52,10 +52,12 @@ class GroupBy extends Projection {
         let node: Node = this.current;
         for (const mapper of this.mappers) {
             const value: any = mapper.value();
-            let child: Node | undefined = node.children.get(value);
+            const key: string =
+                typeof value === "object" && value !== null ? JSON.stringify(value) : String(value);
+            let child: Node | undefined = node.children.get(key);
             if (child === undefined) {
                 child = new Node(value);
-                node.children.set(value, child);
+                node.children.set(key, child);
             }
             node = child;
         }

package/src/parsing/operations/limit.ts

@@ -7,6 +7,15 @@ class Limit extends Operation {
         super();
         this.limit = limit;
     }
+    public get isLimitReached(): boolean {
+        return this.count >= this.limit;
+    }
+    public get limitValue(): number {
+        return this.limit;
+    }
+    public increment(): void {
+        this.count++;
+    }
     public async run(): Promise<void> {
         if (this.count >= this.limit) {
             return;
@@ -19,4 +28,4 @@ class Limit extends Operation {
     }
 }
 
-export default Limit;
+export default Limit;

package/src/parsing/operations/order_by.ts

@@ -0,0 +1,75 @@
+import Operation from "./operation";
+
+export interface SortField {
+    field: string;
+    direction: "asc" | "desc";
+}
+
+/**
+ * Represents an ORDER BY operation that sorts results.
+ *
+ * Can be attached to a RETURN operation (sorting its results),
+ * or used as a standalone accumulating operation after a non-aggregate WITH.
+ *
+ * @example
+ * ```
+ * RETURN x ORDER BY x DESC
+ * ```
+ */
+class OrderBy extends Operation {
+    private _fields: SortField[];
+    private _results: Record<string, any>[] = [];
+
+    constructor(fields: SortField[]) {
+        super();
+        this._fields = fields;
+    }
+
+    public get fields(): SortField[] {
+        return this._fields;
+    }
+
+    /**
+     * Sorts an array of records according to the sort fields.
+     */
+    public sort(records: Record<string, any>[]): Record<string, any>[] {
+        return records.sort((a, b) => {
+            for (const { field, direction } of this._fields) {
+                const aVal = a[field];
+                const bVal = b[field];
+                let cmp = 0;
+                if (aVal == null && bVal == null) cmp = 0;
+                else if (aVal == null) cmp = -1;
+                else if (bVal == null) cmp = 1;
+                else if (aVal < bVal) cmp = -1;
+                else if (aVal > bVal) cmp = 1;
+                if (cmp !== 0) {
+                    return direction === "desc" ? -cmp : cmp;
+                }
+            }
+            return 0;
+        });
+    }
+
+    /**
+     * When used as a standalone operation (after non-aggregate WITH),
+     * accumulates records to sort later.
+     */
+    public async run(): Promise<void> {
+        const record: Record<string, any> = {};
+        // Collect current variable values from the context
+        // This gets called per-row, and then finish() sorts and emits
+        await this.next?.run();
+    }
+
+    public async initialize(): Promise<void> {
+        this._results = [];
+        await this.next?.initialize();
+    }
+
+    public get results(): Record<string, any>[] {
+        return this._results;
+    }
+}
+
+export default OrderBy;

package/src/parsing/operations/return.ts

@@ -1,3 +1,5 @@
+import Limit from "./limit";
+import OrderBy from "./order_by";
 import Projection from "./projection";
 import Where from "./where";
 
@@ -15,6 +17,8 @@ import Where from "./where";
 class Return extends Projection {
     protected _where: Where | null = null;
     protected _results: Record<string, any>[] = [];
+    private _limit: Limit | null = null;
+    protected _orderBy: OrderBy | null = null;
     public set where(where: Where) {
         this._where = where;
     }
@@ -24,10 +28,21 @@ class Return extends Projection {
         }
         return this._where.value();
     }
+    public set limit(limit: Limit) {
+        this._limit = limit;
+    }
+    public set orderBy(orderBy: OrderBy) {
+        this._orderBy = orderBy;
+    }
     public async run(): Promise<void> {
         if (!this.where) {
             return;
         }
+        // When ORDER BY is present, skip limit during accumulation;
+        // limit will be applied after sorting in get results()
+        if (this._orderBy === null && this._limit !== null && this._limit.isLimitReached) {
+            return;
+        }
         const record: Map<string, any> = new Map();
         for (const [expression, alias] of this.expressions()) {
             const raw = expression.value();
@@ -35,12 +50,22 @@ class Return extends Projection {
             record.set(alias, value);
         }
         this._results.push(Object.fromEntries(record));
+        if (this._orderBy === null && this._limit !== null) {
+            this._limit.increment();
+        }
     }
     public async initialize(): Promise<void> {
         this._results = [];
     }
     public get results(): Record<string, any>[] {
-        return this._results;
+        let results = this._results;
+        if (this._orderBy !== null) {
+            results = this._orderBy.sort(results);
+        }
+        if (this._orderBy !== null && this._limit !== null) {
+            results = results.slice(0, this._limit.limitValue);
+        }
+        return results;
     }
 }
 

package/src/parsing/parser.ts

@@ -57,6 +57,7 @@ import Limit from "./operations/limit";
 import Load from "./operations/load";
 import Match from "./operations/match";
 import Operation from "./operations/operation";
+import OrderBy, { SortField } from "./operations/order_by";
 import Return from "./operations/return";
 import Union from "./operations/union";
 import UnionAll from "./operations/union_all";
@@ -112,6 +113,9 @@ class Parser extends BaseParser {
             if (this.token.isUnion()) {
                 break;
             }
+            if (this.token.isEOF()) {
+                break;
+            }
             operation = this.parseOperation();
             if (operation === null && !isSubQuery) {
                 throw new Error("Expected one of WITH, UNWIND, RETURN, LOAD, OR CALL");
@@ -140,10 +144,23 @@ class Parser extends BaseParser {
                     operation = where;
                 }
             }
+            const orderBy = this.parseOrderBy();
+            if (orderBy !== null) {
+                if (operation instanceof Return) {
+                    (operation as Return).orderBy = orderBy;
+                } else {
+                    operation!.addSibling(orderBy);
+                    operation = orderBy;
+                }
+            }
             const limit = this.parseLimit();
             if (limit !== null) {
-                operation!.addSibling(limit);
-                operation = limit;
+                if (operation instanceof Return) {
+                    (operation as Return).limit = limit;
+                } else {
+                    operation!.addSibling(limit);
+                    operation = limit;
+                }
             }
             previous = operation;
         }
@@ -494,16 +511,11 @@ class Parser extends BaseParser {
         node.label = label!;
         if (identifier !== null && this._state.variables.has(identifier)) {
             let reference = this._state.variables.get(identifier);
-            // Resolve through Expression -> Reference -> Node (e.g., after WITH)
-            if (reference instanceof Expression && reference.firstChild() instanceof Reference) {
-                const inner = (reference.firstChild() as Reference).referred;
-                if (inner instanceof Node) {
-                    reference = inner;
-                }
-            }
             if (
                 reference === undefined ||
-                (!(reference instanceof Node) && !(reference instanceof Unwind))
+                (!(reference instanceof Node) &&
+                    !(reference instanceof Unwind) &&
+                    !(reference instanceof Expression))
             ) {
                 throw new Error(`Undefined node reference: ${identifier}`);
             }
@@ -788,6 +800,48 @@ class Parser extends BaseParser {
         return limit;
     }
 
+    private parseOrderBy(): OrderBy | null {
+        this.skipWhitespaceAndComments();
+        if (!this.token.isOrder()) {
+            return null;
+        }
+        this.expectPreviousTokenToBeWhitespaceOrComment();
+        this.setNextToken();
+        this.expectAndSkipWhitespaceAndComments();
+        if (!this.token.isByKeyword()) {
+            throw new Error("Expected BY after ORDER");
+        }
+        this.setNextToken();
+        this.expectAndSkipWhitespaceAndComments();
+        const fields: SortField[] = [];
+        while (true) {
+            if (!this.token.isIdentifierOrKeyword()) {
+                throw new Error("Expected field name in ORDER BY");
+            }
+            const field = this.token.value!;
+            this.setNextToken();
+            this.skipWhitespaceAndComments();
+            let direction: "asc" | "desc" = "asc";
+            if (this.token.isAsc()) {
+                direction = "asc";
+                this.setNextToken();
+                this.skipWhitespaceAndComments();
+            } else if (this.token.isDesc()) {
+                direction = "desc";
+                this.setNextToken();
+                this.skipWhitespaceAndComments();
+            }
+            fields.push({ field, direction });
+            if (this.token.isComma()) {
+                this.setNextToken();
+                this.skipWhitespaceAndComments();
+            } else {
+                break;
+            }
+        }
+        return new OrderBy(fields);
+    }
+
     private *parseExpressions(
         alias_option: AliasOption = AliasOption.NOT_ALLOWED
     ): IterableIterator<Expression> {

package/src/tokenization/token.ts

@@ -675,6 +675,38 @@ class Token {
         return this._type === TokenType.KEYWORD && this._value === Keyword.DISTINCT;
     }
 
+    public static get ORDER(): Token {
+        return new Token(TokenType.KEYWORD, Keyword.ORDER);
+    }
+
+    public isOrder(): boolean {
+        return this._type === TokenType.KEYWORD && this._value === Keyword.ORDER;
+    }
+
+    public static get BY(): Token {
+        return new Token(TokenType.KEYWORD, Keyword.BY);
+    }
+
+    public isByKeyword(): boolean {
+        return this._type === TokenType.KEYWORD && this._value === Keyword.BY;
+    }
+
+    public static get ASC(): Token {
+        return new Token(TokenType.KEYWORD, Keyword.ASC);
+    }
+
+    public isAsc(): boolean {
+        return this._type === TokenType.KEYWORD && this._value === Keyword.ASC;
+    }
+
+    public static get DESC(): Token {
+        return new Token(TokenType.KEYWORD, Keyword.DESC);
+    }
+
+    public isDesc(): boolean {
+        return this._type === TokenType.KEYWORD && this._value === Keyword.DESC;
+    }
+
     public static get LIMIT(): Token {
         return new Token(TokenType.KEYWORD, Keyword.LIMIT);
     }

package/tests/compute/runner.test.ts

@@ -842,6 +842,19 @@ test("Test limit", async () => {
     expect(results.length).toBe(50);
 });
 
+test("Test limit as last operation", async () => {
+    const runner = new Runner(
+        `
+        unwind range(1, 10) as i
+        return i
+        limit 5
+        `
+    );
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(5);
+});
+
 test("Test range lookup", async () => {
     const runner = new Runner(
         `
@@ -1327,6 +1340,60 @@ test("Test match with referenced to previous variable", async () => {
     expect(results[1]).toEqual({ name1: "Person 2", name2: "Person 3", name3: "Person 4" });
 });
 
+test("Test match with aggregated with and subsequent match", async () => {
+    await new Runner(`
+        CREATE VIRTUAL (:User) AS {
+            unwind [
+                {id: 1, name: 'Alice'},
+                {id: 2, name: 'Bob'},
+                {id: 3, name: 'Carol'}
+            ] as record
+            RETURN record.id as id, record.name as name
+        }
+    `).run();
+    await new Runner(`
+        CREATE VIRTUAL (:User)-[:KNOWS]-(:User) AS {
+            unwind [
+                {left_id: 1, right_id: 2},
+                {left_id: 1, right_id: 3}
+            ] as record
+            RETURN record.left_id as left_id, record.right_id as right_id
+        }
+    `).run();
+    await new Runner(`
+        CREATE VIRTUAL (:Project) AS {
+            unwind [
+                {id: 1, name: 'Project A'},
+                {id: 2, name: 'Project B'}
+            ] as record
+            RETURN record.id as id, record.name as name
+        }
+    `).run();
+    await new Runner(`
+        CREATE VIRTUAL (:User)-[:WORKS_ON]-(:Project) AS {
+            unwind [
+                {left_id: 1, right_id: 1},
+                {left_id: 1, right_id: 2}
+            ] as record
+            RETURN record.left_id as left_id, record.right_id as right_id
+        }
+    `).run();
+    const match = new Runner(`
+        MATCH (u:User)-[:KNOWS]->(s:User)
+        WITH u, count(s) as acquaintances
+        MATCH (u)-[:WORKS_ON]->(p:Project)
+        RETURN u.name as name, acquaintances, collect(p.name) as projects
+    `);
+    await match.run();
+    const results = match.results;
+    expect(results.length).toBe(1);
+    expect(results[0]).toEqual({
+        name: "Alice",
+        acquaintances: 2,
+        projects: ["Project A", "Project B"],
+    });
+});
+
 test("Test match and return full node", async () => {
     await new Runner(`
         CREATE VIRTUAL (:Person) AS {
@@ -2481,6 +2548,54 @@ test("Test WHERE with IN combined with AND", async () => {
     expect(results.map((r: any) => r.n)).toEqual([10, 15, 20]);
 });
 
+test("Test WHERE with AND before IN", async () => {
+    const runner = new Runner(`
+        unwind ['expert', 'intermediate', 'beginner'] as proficiency
+        with proficiency where 1=1 and proficiency in ['expert']
+        return proficiency
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(1);
+    expect(results[0]).toEqual({ proficiency: "expert" });
+});
+
+test("Test WHERE with AND before NOT IN", async () => {
+    const runner = new Runner(`
+        unwind ['expert', 'intermediate', 'beginner'] as proficiency
+        with proficiency where 1=1 and proficiency not in ['expert']
+        return proficiency
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(2);
+    expect(results.map((r: any) => r.proficiency)).toEqual(["intermediate", "beginner"]);
+});
+
+test("Test WHERE with OR before IN", async () => {
+    const runner = new Runner(`
+        unwind range(1, 10) as n
+        with n where 1=0 or n in [3, 7]
+        return n
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(2);
+    expect(results.map((r: any) => r.n)).toEqual([3, 7]);
+});
+
+test("Test IN as return expression with AND in WHERE", async () => {
+    const runner = new Runner(`
+        unwind ['expert', 'intermediate', 'beginner'] as proficiency
+        with proficiency where 1=1 and proficiency in ['expert']
+        return proficiency, proficiency in ['expert'] as isExpert
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(1);
+    expect(results[0]).toEqual({ proficiency: "expert", isExpert: 1 });
+});
+
 test("Test WHERE with CONTAINS", async () => {
     const runner = new Runner(`
         unwind ['apple', 'banana', 'grape', 'pineapple'] as fruit
@@ -3692,3 +3807,99 @@ test("Test duration() with time only", async () => {
     expect(d.totalSeconds).toBe(9000);
     expect(d.formatted).toBe("PT2H30M");
 });
+
+// ORDER BY tests
+
+test("Test order by ascending", async () => {
+    const runner = new Runner("unwind [3, 1, 2] as x return x order by x");
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ x: 1 });
+    expect(results[1]).toEqual({ x: 2 });
+    expect(results[2]).toEqual({ x: 3 });
+});
+
+test("Test order by descending", async () => {
+    const runner = new Runner("unwind [3, 1, 2] as x return x order by x desc");
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ x: 3 });
+    expect(results[1]).toEqual({ x: 2 });
+    expect(results[2]).toEqual({ x: 1 });
+});
+
+test("Test order by ascending explicit", async () => {
+    const runner = new Runner("unwind [3, 1, 2] as x return x order by x asc");
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ x: 1 });
+    expect(results[1]).toEqual({ x: 2 });
+    expect(results[2]).toEqual({ x: 3 });
+});
+
+test("Test order by with multiple fields", async () => {
+    const runner = new Runner(`
+        unwind [{name: 'Alice', age: 30}, {name: 'Bob', age: 25}, {name: 'Alice', age: 25}] as person
+        return person.name as name, person.age as age
+        order by name asc, age asc
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ name: "Alice", age: 25 });
+    expect(results[1]).toEqual({ name: "Alice", age: 30 });
+    expect(results[2]).toEqual({ name: "Bob", age: 25 });
+});
+
+test("Test order by with strings", async () => {
+    const runner = new Runner(
+        "unwind ['banana', 'apple', 'cherry'] as fruit return fruit order by fruit"
+    );
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ fruit: "apple" });
+    expect(results[1]).toEqual({ fruit: "banana" });
+    expect(results[2]).toEqual({ fruit: "cherry" });
+});
+
+test("Test order by with aggregated return", async () => {
+    const runner = new Runner(`
+        unwind [1, 1, 2, 2, 3, 3] as x
+        return x, count(x) as cnt
+        order by x desc
+    `);
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ x: 3, cnt: 2 });
+    expect(results[1]).toEqual({ x: 2, cnt: 2 });
+    expect(results[2]).toEqual({ x: 1, cnt: 2 });
+});
+
+test("Test order by with limit", async () => {
+    const runner = new Runner("unwind [3, 1, 4, 1, 5, 9, 2, 6] as x return x order by x limit 3");
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(3);
+    expect(results[0]).toEqual({ x: 1 });
+    expect(results[1]).toEqual({ x: 1 });
+    expect(results[2]).toEqual({ x: 2 });
+});
+
+test("Test order by with where", async () => {
+    const runner = new Runner(
+        "unwind [3, 1, 4, 1, 5, 9, 2, 6] as x return x where x > 2 order by x desc"
+    );
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(5);
+    expect(results[0]).toEqual({ x: 9 });
+    expect(results[1]).toEqual({ x: 6 });
+    expect(results[2]).toEqual({ x: 5 });
+    expect(results[3]).toEqual({ x: 4 });
+    expect(results[4]).toEqual({ x: 3 });
+});