flowquery 1.0.15 → 1.0.17

package/misc/apps/RAG/src/prompts/FlowQuerySystemPrompt.ts

@@ -1,15 +1,15 @@
 /**
  * FlowQuery System Prompt Generator
- * 
+ *
  * Generates a system prompt that instructs the LLM to create FlowQuery statements
  * based on natural language queries, with awareness of available loader plugins.
- * 
+ *
  * Uses FlowQuery's built-in functions() introspection to dynamically discover
  * available async data loaders and their metadata.
  */
+import { FunctionMetadata, OutputSchema, ParameterSchema } from "flowquery/extensibility";
 
-import { FunctionMetadata, ParameterSchema, OutputSchema } from 'flowquery/extensibility';
-import { getAllPluginMetadata, getAvailableLoaders } from '../plugins';
+import { getAllPluginMetadata, getAvailableLoaders } from "../plugins";
 
 /**
  * FlowQuery language reference documentation.
@@ -28,25 +28,29 @@ FlowQuery is a declarative query language for data processing pipelines. It uses
    WITH expression1 AS var1, expression2 AS var2
    \`\`\`
 
-2. **LOAD JSON FROM** - Load data from a URL or async data provider
+2. **LOAD JSON FROM** - Load data from a URL
    \`\`\`
    LOAD JSON FROM 'https://api.example.com/data' AS item
-   LOAD JSON FROM myFunction(arg1, arg2) AS item
+   \`\`\`
+
+3. **CALL ... YIELD** - Call an async data provider function and yield its fields
+   \`\`\`
+   CALL myFunction(arg1, arg2) YIELD field1, field2, field3
    \`\`\`
    
-   **IMPORTANT**: Async data providers (functions used after LOAD JSON FROM) cannot be nested inside other function calls. If you need to pass data from one async provider to another, first load the data into a variable using collect(), then pass that variable:
+   **IMPORTANT**: Async data providers (functions used with CALL) cannot be nested inside other function calls. If you need to pass data from one async provider to another, first load the data into a variable using collect(), then pass that variable:
    \`\`\`
    // WRONG - async providers cannot be nested:
-   // LOAD JSON FROM table(mockProducts(5), 'Products') AS card
+   // CALL table(mockProducts(5), 'Products') YIELD html
    
    // CORRECT - collect data first, then pass to next provider:
-   LOAD JSON FROM mockProducts(5) AS p
-   WITH collect(p) AS products
-   LOAD JSON FROM table(products, 'Products') AS card
-   RETURN card
+   CALL mockProducts(5) YIELD id, name, price
+   WITH collect({ id: id, name: name, price: price }) AS products
+   CALL table(products, 'Products') YIELD html
+   RETURN html
    \`\`\`
 
-3. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
+4. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
    \`\`\`
    LOAD JSON FROM 'https://api.example.com/data'
    HEADERS {
@@ -59,42 +63,38 @@ FlowQuery is a declarative query language for data processing pipelines. It uses
    } AS response
    \`\`\`
 
-4. **UNWIND** - Expand arrays into individual rows
+5. **UNWIND** - Expand arrays into individual rows
    \`\`\`
    UNWIND [1, 2, 3] AS number
    UNWIND myArray AS item
    UNWIND range(0, 10) AS index
    \`\`\`
+   
+   **IMPORTANT**: An UNWIND statement cannot be followed directly by a WHERE statement. If you need to filter after unwinding, use a WITH clause in between:
+   \`\`\`
+   // WRONG - UNWIND cannot be directly followed by WHERE:
+   // UNWIND items AS item
+   // WHERE item.active = true
+   
+   // CORRECT - use WITH between UNWIND and WHERE:
+   UNWIND items AS item
+   WITH item
+   WHERE item.active = true
+   \`\`\`
 
-5. **WHERE** - Filter results
+6. **WHERE** - Filter results
    \`\`\`
    WHERE item.active = true
    WHERE user.age > 18 AND user.name CONTAINS 'John'
    \`\`\`
 
-6. **RETURN** - Specify output columns
+7. **RETURN** - Specify output columns
    \`\`\`
    RETURN item.name, item.value
    RETURN item.name AS Name, item.price AS Price
    RETURN *  -- Return all fields
    \`\`\`
 
-7. **ORDER BY** - Sort results
-   \`\`\`
-   ORDER BY item.name ASC
-   ORDER BY item.price DESC, item.name ASC
-   \`\`\`
-
-8. **LIMIT** - Limit number of results
-   \`\`\`
-   LIMIT 10
-   \`\`\`
-
-9. **SKIP** - Skip a number of results
-   \`\`\`
-   SKIP 5
-   \`\`\`
-
 ### Built-in Functions
 
 - **String Functions**: \`size()\`, \`substring()\`, \`trim()\`, \`toLower()\`, \`toUpper()\`, \`split()\`, \`join()\`, \`replace()\`, \`startsWith()\`, \`endsWith()\`, \`contains()\`
@@ -154,9 +154,12 @@ export class FlowQuerySystemPrompt {
      * Format a parameter schema into a readable string.
      */
     private static formatParameter(param: ParameterSchema): string {
-        const required = param.required ? ' (required)' : ' (optional)';
-        const defaultVal = param.default !== undefined ? `, default: ${JSON.stringify(param.default)}` : '';
-        const enumVals = param.enum ? `, values: [${param.enum.map(v => JSON.stringify(v)).join(', ')}]` : '';
+        const required = param.required ? " (required)" : " (optional)";
+        const defaultVal =
+            param.default !== undefined ? `, default: ${JSON.stringify(param.default)}` : "";
+        const enumVals = param.enum
+            ? `, values: [${param.enum.map((v) => JSON.stringify(v)).join(", ")}]`
+            : "";
         return `  - \`${param.name}\`: ${param.type}${required}${defaultVal}${enumVals} - ${param.description}`;
     }
 
@@ -165,18 +168,18 @@ export class FlowQuerySystemPrompt {
      */
     private static formatOutput(output: OutputSchema): string {
         let result = `  Returns: ${output.type} - ${output.description}`;
-        
+
         if (output.properties) {
-            result += '\n  Output properties:';
+            result += "\n  Output properties:";
             for (const [key, prop] of Object.entries(output.properties)) {
                 result += `\n    - \`${key}\`: ${prop.type} - ${prop.description}`;
             }
         }
-        
+
         if (output.example) {
             result += `\n  Example output: ${JSON.stringify(output.example, null, 2)}`;
         }
-        
+
         return result;
     }
 
@@ -185,38 +188,38 @@ export class FlowQuerySystemPrompt {
      */
     private static formatPluginDocumentation(plugin: FunctionMetadata): string {
         const lines: string[] = [];
-        
+
         lines.push(`### \`${plugin.name}\``);
         lines.push(`**Description**: ${plugin.description}`);
-        
+
         if (plugin.category) {
             lines.push(`**Category**: ${plugin.category}`);
         }
-        
+
         if (plugin.parameters.length > 0) {
-            lines.push('\n**Parameters**:');
+            lines.push("\n**Parameters**:");
             for (const param of plugin.parameters) {
                 lines.push(this.formatParameter(param));
             }
         } else {
-            lines.push('\n**Parameters**: None');
+            lines.push("\n**Parameters**: None");
         }
-        
-        lines.push('\n**Output**:');
+
+        lines.push("\n**Output**:");
         lines.push(this.formatOutput(plugin.output));
-        
+
         if (plugin.examples && plugin.examples.length > 0) {
-            lines.push('\n**Usage Examples**:');
+            lines.push("\n**Usage Examples**:");
             for (const example of plugin.examples) {
                 lines.push(`\`\`\`\n${example}\n\`\`\``);
             }
         }
-        
+
         if (plugin.notes) {
             lines.push(`\n**Notes**: ${plugin.notes}`);
         }
-        
-        return lines.join('\n');
+
+        return lines.join("\n");
     }
 
     /**
@@ -224,33 +227,37 @@ export class FlowQuerySystemPrompt {
      */
     private static generatePluginDocumentation(plugins: FunctionMetadata[]): string {
         if (plugins.length === 0) {
-            return 'No data loader plugins are currently available.';
+            return "No data loader plugins are currently available.";
         }
-        
+
         const sections: string[] = [];
-        
+
         // Group plugins by category
         const byCategory = new Map<string, FunctionMetadata[]>();
         for (const plugin of plugins) {
-            const category = plugin.category || 'general';
+            const category = plugin.category || "general";
             if (!byCategory.has(category)) {
                 byCategory.set(category, []);
             }
             byCategory.get(category)!.push(plugin);
         }
-        
-        sections.push('## Available Data Loader Plugins\n');
-        sections.push('The following async data loader functions are available for use with `LOAD JSON FROM`:\n');
-        
+
+        sections.push("## Available Data Loader Plugins\n");
+        sections.push(
+            "The following async data loader functions are available for use with `CALL ... YIELD`:\n"
+        );
+
         for (const [category, categoryPlugins] of byCategory) {
-            sections.push(`\n### Category: ${category.charAt(0).toUpperCase() + category.slice(1)}\n`);
+            sections.push(
+                `\n### Category: ${category.charAt(0).toUpperCase() + category.slice(1)}\n`
+            );
             for (const plugin of categoryPlugins) {
                 sections.push(this.formatPluginDocumentation(plugin));
-                sections.push('---');
+                sections.push("---");
             }
         }
-        
-        return sections.join('\n');
+
+        return sections.join("\n");
     }
 
     /**
@@ -296,15 +303,15 @@ ${FLOWQUERY_LANGUAGE_REFERENCE}
 
 ${pluginDocs}
 
-${additionalContext ? `## Additional Context\n\n${additionalContext}` : ''}
+${additionalContext ? `## Additional Context\n\n${additionalContext}` : ""}
 
 ## Example Response Format
 
 **When a query is needed**:
 \`\`\`flowquery
-LOAD JSON FROM pluginName(args) AS item
-WHERE item.field = 'value'
-RETURN item.name AS Name, item.value AS Value
+CALL pluginName(args) YIELD field1, field2, field3
+WHERE field1 = 'value'
+RETURN field1 AS Name, field2 AS Value
 \`\`\`
 
 **When no query is needed** (e.g., general questions about FlowQuery):
@@ -317,7 +324,7 @@ Now help the user with their request.`;
     /**
      * Generate the complete FlowQuery system prompt.
      * Uses FlowQuery's introspection via functions() as the single source of truth.
-     * 
+     *
      * @param additionalContext - Optional additional context to include in the prompt
      * @returns The complete system prompt string
      */
@@ -325,14 +332,14 @@ Now help the user with their request.`;
         // Uses FlowQuery's introspection to get available async providers
         const plugins = getAllPluginMetadata();
         const pluginDocs = this.generatePluginDocumentation(plugins);
-        
+
         return this.buildSystemPrompt(pluginDocs, additionalContext);
     }
 
     /**
      * Generate a system prompt for the interpretation phase.
      * Used after FlowQuery execution to interpret results.
-     * 
+     *
      * @returns The interpretation system prompt string
      */
     public static generateInterpretationPrompt(): string {
@@ -363,8 +370,8 @@ You are now receiving the execution results. Your job is to:
      */
     public static getMinimalPrompt(): string {
         const plugins = getAllPluginMetadata();
-        const pluginList = plugins.map(p => `- \`${p.name}\`: ${p.description}`).join('\n');
-        
+        const pluginList = plugins.map((p) => `- \`${p.name}\`: ${p.description}`).join("\n");
+
         return `You are a FlowQuery assistant. Generate FlowQuery statements based on user requests.
 
 Available data loader plugins:
@@ -379,7 +386,7 @@ Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
     /**
      * Generate the FlowQuery system prompt asynchronously using functions() introspection.
      * This is the preferred method that uses FlowQuery's built-in introspection.
-     * 
+     *
      * @param additionalContext - Optional additional context to include in the prompt
      * @returns Promise resolving to the complete system prompt string
      */
@@ -387,15 +394,19 @@ Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
         // Use FlowQuery's functions() introspection to discover available loaders
         const plugins = await getAvailableLoaders();
         const pluginDocs = this.generatePluginDocumentation(plugins);
-        
+
         return this.buildSystemPrompt(pluginDocs, additionalContext);
     }
 }
 
 // Export functions for backward compatibility
-export const generateFlowQuerySystemPrompt = FlowQuerySystemPrompt.generate.bind(FlowQuerySystemPrompt);
-export const generateInterpretationPrompt = FlowQuerySystemPrompt.generateInterpretationPrompt.bind(FlowQuerySystemPrompt);
-export const getMinimalFlowQueryPrompt = FlowQuerySystemPrompt.getMinimalPrompt.bind(FlowQuerySystemPrompt);
-export const generateFlowQuerySystemPromptAsync = FlowQuerySystemPrompt.generateAsync.bind(FlowQuerySystemPrompt);
+export const generateFlowQuerySystemPrompt =
+    FlowQuerySystemPrompt.generate.bind(FlowQuerySystemPrompt);
+export const generateInterpretationPrompt =
+    FlowQuerySystemPrompt.generateInterpretationPrompt.bind(FlowQuerySystemPrompt);
+export const getMinimalFlowQueryPrompt =
+    FlowQuerySystemPrompt.getMinimalPrompt.bind(FlowQuerySystemPrompt);
+export const generateFlowQuerySystemPromptAsync =
+    FlowQuerySystemPrompt.generateAsync.bind(FlowQuerySystemPrompt);
 
 export default FlowQuerySystemPrompt;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",

package/src/compute/runner.ts

@@ -1,14 +1,15 @@
+import ASTNode from "../parsing/ast_node";
+import Function from "../parsing/functions/function";
+import { FunctionMetadata } from "../parsing/functions/function_metadata";
 import Operation from "../parsing/operations/operation";
 import Parser from "../parsing/parser";
-import { FunctionMetadata } from "../parsing/functions/function_metadata";
-import Function from "../parsing/functions/function";
 
 /**
  * Executes a FlowQuery statement and retrieves the results.
- * 
+ *
  * The Runner class parses a FlowQuery statement into an AST and executes it,
  * managing the execution flow from the first operation to the final return statement.
- * 
+ *
  * @example
  * ```typescript
  * const runner = new Runner("WITH 1 as x RETURN x");
@@ -24,7 +25,11 @@ class Runner {
      * List all registered functions with their metadata.
      * Added dynamically in index.browser.ts / index.node.ts
      */
-    static listFunctions: (options?: { category?: string; asyncOnly?: boolean; syncOnly?: boolean }) => FunctionMetadata[];
+    static listFunctions: (options?: {
+        category?: string;
+        asyncOnly?: boolean;
+        syncOnly?: boolean;
+    }) => FunctionMetadata[];
 
     /**
      * Get metadata for a specific function.
@@ -37,32 +42,32 @@ class Runner {
      * Added dynamically in index.browser.ts / index.node.ts
      */
     static Function: typeof Function;
-    
+
     /**
      * Creates a new Runner instance and parses the FlowQuery statement.
-     * 
+     *
      * @param statement - The FlowQuery statement to execute
      * @throws {Error} If the statement is null, empty, or contains syntax errors
      */
-    constructor(statement: string | null = null) {
-        if(statement === null || statement === "") {
-            throw new Error("Statement cannot be null or empty");
+    constructor(statement: string | null = null, ast: ASTNode | null = null) {
+        if ((statement === null || statement === "") && ast === null) {
+            throw new Error("Either statement or AST must be provided");
         }
-        const parser = new Parser();
-        const ast = parser.parse(statement);
-        this.first = ast.firstChild() as Operation;
-        this.last = ast.lastChild() as Operation;
+        const _ast = ast !== null ? ast : new Parser().parse(statement!);
+        this.first = _ast.firstChild() as Operation;
+        this.last = _ast.lastChild() as Operation;
     }
-    
+
     /**
      * Executes the parsed FlowQuery statement.
-     * 
+     *
      * @returns A promise that resolves when execution completes
      * @throws {Error} If an error occurs during execution
      */
     public async run(): Promise<void> {
         return new Promise<void>(async (resolve, reject) => {
             try {
+                await this.first.initialize();
                 await this.first.run();
                 await this.first.finish();
                 resolve();
@@ -71,10 +76,10 @@ class Runner {
             }
         });
     }
-    
+
     /**
      * Gets the results from the executed statement.
-     * 
+     *
      * @returns The results from the last operation (typically a RETURN statement)
      */
     public get results(): any {
@@ -82,4 +87,4 @@ class Runner {
     }
 }
 
-export default Runner;
+export default Runner;

package/src/graph/data.ts

@@ -0,0 +1,112 @@
+class IndexEntry {
+    private _positions: number[];
+    private _index: number = -1;
+
+    constructor(positions: number[] = []) {
+        this._positions = positions;
+    }
+    public add(position: number): void {
+        this._positions.push(position);
+    }
+    public get position(): number {
+        return this._positions[this._index];
+    }
+    public reset(): void {
+        this._index = -1;
+    }
+    public next(): boolean {
+        if (this._index < this._positions.length - 1) {
+            this._index++;
+            return true;
+        }
+        return false;
+    }
+    public clone(): IndexEntry {
+        return new IndexEntry([...this._positions]);
+    }
+}
+
+class Layer {
+    private _index: Map<string, IndexEntry> = new Map();
+    private _current: number = -1;
+    constructor(index: Map<string, IndexEntry>) {
+        this._index = index;
+    }
+    public get index(): Map<string, IndexEntry> {
+        return this._index;
+    }
+    public get current(): number {
+        return this._current;
+    }
+    public set current(value: number) {
+        this._current = value;
+    }
+}
+
+class Data {
+    protected _records: Record<string, any>[] = [];
+    private _layers: Map<number, Layer> = new Map();
+
+    constructor(records: Record<string, any>[] = []) {
+        this._records = records;
+        this._layers.set(0, new Layer(new Map()));
+    }
+    protected _buildIndex(key: string, level: number = 0): void {
+        this.layer(level).index.clear();
+        this._records.forEach((record, idx) => {
+            if (record.hasOwnProperty(key)) {
+                if (!this.layer(level).index.has(record[key])) {
+                    this.layer(level).index.set(record[key], new IndexEntry());
+                }
+                this.layer(level).index.get(record[key])!.add(idx);
+            }
+        });
+    }
+    public layer(level: number = 0): Layer {
+        if (!this._layers.has(level)) {
+            const first = this._layers.get(0)!;
+            const cloned = new Map<string, IndexEntry>();
+            for (const [key, entry] of first.index) {
+                cloned.set(key, entry.clone());
+            }
+            this._layers.set(level, new Layer(cloned));
+        }
+        return this._layers.get(level)!;
+    }
+    protected _find(key: string, level: number = 0): boolean {
+        if (!this.layer(level).index.has(key)) {
+            this.layer(level).current = this._records.length; // Move to end
+            return false;
+        } else {
+            const entry = this.layer(level).index.get(key)!;
+            const more = entry.next();
+            if (!more) {
+                this.layer(level).current = this._records.length; // Move to end
+                return false;
+            }
+            this.layer(level).current = entry.position;
+            return true;
+        }
+    }
+    public reset(): void {
+        this.layer(0).current = -1;
+        for (const entry of this.layer(0).index.values()) {
+            entry.reset();
+        }
+    }
+    public next(level: number = 0): boolean {
+        if (this.layer(level).current < this._records.length - 1) {
+            this.layer(level).current++;
+            return true;
+        }
+        return false;
+    }
+    public current(level: number = 0): Record<string, any> | null {
+        if (this.layer(level).current < this._records.length) {
+            return this._records[this.layer(level).current];
+        }
+        return null;
+    }
+}
+
+export default Data;

package/src/graph/database.ts

@@ -0,0 +1,63 @@
+import ASTNode from "../parsing/ast_node";
+import Node from "./node";
+import NodeData, { NodeRecord } from "./node_data";
+import PhysicalNode from "./physical_node";
+import PhysicalRelationship from "./physical_relationship";
+import Relationship from "./relationship";
+import RelationshipData, { RelationshipRecord } from "./relationship_data";
+
+class Database {
+    private static instance: Database;
+    private static nodes: Map<string, PhysicalNode> = new Map();
+    private static relationships: Map<string, PhysicalRelationship> = new Map();
+
+    public static getInstance(): Database {
+        if (!Database.instance) {
+            Database.instance = new Database();
+        }
+        return Database.instance;
+    }
+    public addNode(node: Node, statement: ASTNode): void {
+        if (node.label === null) {
+            throw new Error("Node label is null");
+        }
+        const physical = new PhysicalNode(null, node.label);
+        physical.statement = statement;
+        Database.nodes.set(node.label, physical);
+    }
+    public getNode(node: Node): PhysicalNode | null {
+        return Database.nodes.get(node.label!) || null;
+    }
+    public addRelationship(relationship: Relationship, statement: ASTNode): void {
+        if (relationship.type === null) {
+            throw new Error("Relationship type is null");
+        }
+        const physical = new PhysicalRelationship(null, relationship.type);
+        physical.statement = statement;
+        Database.relationships.set(relationship.type, physical);
+    }
+    public getRelationship(relationship: Relationship): PhysicalRelationship | null {
+        return Database.relationships.get(relationship.type!) || null;
+    }
+    public async getData(element: Node | Relationship): Promise<NodeData | RelationshipData> {
+        if (element instanceof Node) {
+            const node = this.getNode(element);
+            if (node === null) {
+                throw new Error(`Physical node not found for label ${element.label}`);
+            }
+            const data = await node.data();
+            return new NodeData(data as NodeRecord[]);
+        } else if (element instanceof Relationship) {
+            const relationship = this.getRelationship(element);
+            if (relationship === null) {
+                throw new Error(`Physical relationship not found for type ${element.type}`);
+            }
+            const data = await relationship.data();
+            return new RelationshipData(data as RelationshipRecord[]);
+        } else {
+            throw new Error("Element is neither Node nor Relationship");
+        }
+    }
+}
+
+export default Database;

package/src/graph/hops.ts

@@ -0,0 +1,22 @@
+class Hops {
+    private _min: number = 0;
+    private _max: number = 1;
+
+    public set min(min: number) {
+        this._min = min;
+    }
+    public get min(): number {
+        return this._min;
+    }
+    public set max(max: number) {
+        this._max = max;
+    }
+    public get max(): number {
+        return this._max;
+    }
+    public multi(): boolean {
+        return this._max > 1 || this._max === -1;
+    }
+}
+
+export default Hops;