flowquery 1.0.15 → 1.0.16

package/misc/apps/RAG/src/prompts/FlowQuerySystemPrompt.ts

@@ -1,15 +1,15 @@
 /**
  * FlowQuery System Prompt Generator
- * 
+ *
  * Generates a system prompt that instructs the LLM to create FlowQuery statements
  * based on natural language queries, with awareness of available loader plugins.
- * 
+ *
  * Uses FlowQuery's built-in functions() introspection to dynamically discover
  * available async data loaders and their metadata.
  */
+import { FunctionMetadata, OutputSchema, ParameterSchema } from "flowquery/extensibility";
 
-import { FunctionMetadata, ParameterSchema, OutputSchema } from 'flowquery/extensibility';
-import { getAllPluginMetadata, getAvailableLoaders } from '../plugins';
+import { getAllPluginMetadata, getAvailableLoaders } from "../plugins";
 
 /**
  * FlowQuery language reference documentation.
@@ -28,25 +28,29 @@ FlowQuery is a declarative query language for data processing pipelines. It uses
    WITH expression1 AS var1, expression2 AS var2
    \`\`\`
 
-2. **LOAD JSON FROM** - Load data from a URL or async data provider
+2. **LOAD JSON FROM** - Load data from a URL
    \`\`\`
    LOAD JSON FROM 'https://api.example.com/data' AS item
-   LOAD JSON FROM myFunction(arg1, arg2) AS item
+   \`\`\`
+
+3. **CALL ... YIELD** - Call an async data provider function and yield its fields
+   \`\`\`
+   CALL myFunction(arg1, arg2) YIELD field1, field2, field3
    \`\`\`
    
-   **IMPORTANT**: Async data providers (functions used after LOAD JSON FROM) cannot be nested inside other function calls. If you need to pass data from one async provider to another, first load the data into a variable using collect(), then pass that variable:
+   **IMPORTANT**: Async data providers (functions used with CALL) cannot be nested inside other function calls. If you need to pass data from one async provider to another, first load the data into a variable using collect(), then pass that variable:
    \`\`\`
    // WRONG - async providers cannot be nested:
-   // LOAD JSON FROM table(mockProducts(5), 'Products') AS card
+   // CALL table(mockProducts(5), 'Products') YIELD html
    
    // CORRECT - collect data first, then pass to next provider:
-   LOAD JSON FROM mockProducts(5) AS p
-   WITH collect(p) AS products
-   LOAD JSON FROM table(products, 'Products') AS card
-   RETURN card
+   CALL mockProducts(5) YIELD id, name, price
+   WITH collect({ id: id, name: name, price: price }) AS products
+   CALL table(products, 'Products') YIELD html
+   RETURN html
    \`\`\`
 
-3. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
+4. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
    \`\`\`
    LOAD JSON FROM 'https://api.example.com/data'
    HEADERS {
@@ -59,42 +63,26 @@ FlowQuery is a declarative query language for data processing pipelines. It uses
    } AS response
    \`\`\`
 
-4. **UNWIND** - Expand arrays into individual rows
+5. **UNWIND** - Expand arrays into individual rows
    \`\`\`
    UNWIND [1, 2, 3] AS number
    UNWIND myArray AS item
    UNWIND range(0, 10) AS index
    \`\`\`
 
-5. **WHERE** - Filter results
+6. **WHERE** - Filter results
    \`\`\`
    WHERE item.active = true
    WHERE user.age > 18 AND user.name CONTAINS 'John'
    \`\`\`
 
-6. **RETURN** - Specify output columns
+7. **RETURN** - Specify output columns
    \`\`\`
    RETURN item.name, item.value
    RETURN item.name AS Name, item.price AS Price
    RETURN *  -- Return all fields
    \`\`\`
 
-7. **ORDER BY** - Sort results
-   \`\`\`
-   ORDER BY item.name ASC
-   ORDER BY item.price DESC, item.name ASC
-   \`\`\`
-
-8. **LIMIT** - Limit number of results
-   \`\`\`
-   LIMIT 10
-   \`\`\`
-
-9. **SKIP** - Skip a number of results
-   \`\`\`
-   SKIP 5
-   \`\`\`
-
 ### Built-in Functions
 
 - **String Functions**: \`size()\`, \`substring()\`, \`trim()\`, \`toLower()\`, \`toUpper()\`, \`split()\`, \`join()\`, \`replace()\`, \`startsWith()\`, \`endsWith()\`, \`contains()\`
@@ -154,9 +142,12 @@ export class FlowQuerySystemPrompt {
      * Format a parameter schema into a readable string.
      */
     private static formatParameter(param: ParameterSchema): string {
-        const required = param.required ? ' (required)' : ' (optional)';
-        const defaultVal = param.default !== undefined ? `, default: ${JSON.stringify(param.default)}` : '';
-        const enumVals = param.enum ? `, values: [${param.enum.map(v => JSON.stringify(v)).join(', ')}]` : '';
+        const required = param.required ? " (required)" : " (optional)";
+        const defaultVal =
+            param.default !== undefined ? `, default: ${JSON.stringify(param.default)}` : "";
+        const enumVals = param.enum
+            ? `, values: [${param.enum.map((v) => JSON.stringify(v)).join(", ")}]`
+            : "";
         return `  - \`${param.name}\`: ${param.type}${required}${defaultVal}${enumVals} - ${param.description}`;
     }
 
@@ -165,18 +156,18 @@ export class FlowQuerySystemPrompt {
      */
     private static formatOutput(output: OutputSchema): string {
         let result = `  Returns: ${output.type} - ${output.description}`;
-        
+
         if (output.properties) {
-            result += '\n  Output properties:';
+            result += "\n  Output properties:";
             for (const [key, prop] of Object.entries(output.properties)) {
                 result += `\n    - \`${key}\`: ${prop.type} - ${prop.description}`;
             }
         }
-        
+
         if (output.example) {
             result += `\n  Example output: ${JSON.stringify(output.example, null, 2)}`;
         }
-        
+
         return result;
     }
 
@@ -185,38 +176,38 @@ export class FlowQuerySystemPrompt {
      */
     private static formatPluginDocumentation(plugin: FunctionMetadata): string {
         const lines: string[] = [];
-        
+
         lines.push(`### \`${plugin.name}\``);
         lines.push(`**Description**: ${plugin.description}`);
-        
+
         if (plugin.category) {
             lines.push(`**Category**: ${plugin.category}`);
         }
-        
+
         if (plugin.parameters.length > 0) {
-            lines.push('\n**Parameters**:');
+            lines.push("\n**Parameters**:");
             for (const param of plugin.parameters) {
                 lines.push(this.formatParameter(param));
             }
         } else {
-            lines.push('\n**Parameters**: None');
+            lines.push("\n**Parameters**: None");
         }
-        
-        lines.push('\n**Output**:');
+
+        lines.push("\n**Output**:");
         lines.push(this.formatOutput(plugin.output));
-        
+
         if (plugin.examples && plugin.examples.length > 0) {
-            lines.push('\n**Usage Examples**:');
+            lines.push("\n**Usage Examples**:");
             for (const example of plugin.examples) {
                 lines.push(`\`\`\`\n${example}\n\`\`\``);
             }
         }
-        
+
         if (plugin.notes) {
             lines.push(`\n**Notes**: ${plugin.notes}`);
         }
-        
-        return lines.join('\n');
+
+        return lines.join("\n");
     }
 
     /**
@@ -224,33 +215,37 @@ export class FlowQuerySystemPrompt {
      */
     private static generatePluginDocumentation(plugins: FunctionMetadata[]): string {
         if (plugins.length === 0) {
-            return 'No data loader plugins are currently available.';
+            return "No data loader plugins are currently available.";
         }
-        
+
         const sections: string[] = [];
-        
+
         // Group plugins by category
         const byCategory = new Map<string, FunctionMetadata[]>();
         for (const plugin of plugins) {
-            const category = plugin.category || 'general';
+            const category = plugin.category || "general";
             if (!byCategory.has(category)) {
                 byCategory.set(category, []);
             }
             byCategory.get(category)!.push(plugin);
         }
-        
-        sections.push('## Available Data Loader Plugins\n');
-        sections.push('The following async data loader functions are available for use with `LOAD JSON FROM`:\n');
-        
+
+        sections.push("## Available Data Loader Plugins\n");
+        sections.push(
+            "The following async data loader functions are available for use with `CALL ... YIELD`:\n"
+        );
+
         for (const [category, categoryPlugins] of byCategory) {
-            sections.push(`\n### Category: ${category.charAt(0).toUpperCase() + category.slice(1)}\n`);
+            sections.push(
+                `\n### Category: ${category.charAt(0).toUpperCase() + category.slice(1)}\n`
+            );
             for (const plugin of categoryPlugins) {
                 sections.push(this.formatPluginDocumentation(plugin));
-                sections.push('---');
+                sections.push("---");
             }
         }
-        
-        return sections.join('\n');
+
+        return sections.join("\n");
     }
 
     /**
@@ -296,15 +291,15 @@ ${FLOWQUERY_LANGUAGE_REFERENCE}
 
 ${pluginDocs}
 
-${additionalContext ? `## Additional Context\n\n${additionalContext}` : ''}
+${additionalContext ? `## Additional Context\n\n${additionalContext}` : ""}
 
 ## Example Response Format
 
 **When a query is needed**:
 \`\`\`flowquery
-LOAD JSON FROM pluginName(args) AS item
-WHERE item.field = 'value'
-RETURN item.name AS Name, item.value AS Value
+CALL pluginName(args) YIELD field1, field2, field3
+WHERE field1 = 'value'
+RETURN field1 AS Name, field2 AS Value
 \`\`\`
 
 **When no query is needed** (e.g., general questions about FlowQuery):
@@ -317,7 +312,7 @@ Now help the user with their request.`;
     /**
      * Generate the complete FlowQuery system prompt.
      * Uses FlowQuery's introspection via functions() as the single source of truth.
-     * 
+     *
      * @param additionalContext - Optional additional context to include in the prompt
      * @returns The complete system prompt string
      */
@@ -325,14 +320,14 @@ Now help the user with their request.`;
         // Uses FlowQuery's introspection to get available async providers
         const plugins = getAllPluginMetadata();
         const pluginDocs = this.generatePluginDocumentation(plugins);
-        
+
         return this.buildSystemPrompt(pluginDocs, additionalContext);
     }
 
     /**
      * Generate a system prompt for the interpretation phase.
      * Used after FlowQuery execution to interpret results.
-     * 
+     *
      * @returns The interpretation system prompt string
      */
     public static generateInterpretationPrompt(): string {
@@ -363,8 +358,8 @@ You are now receiving the execution results. Your job is to:
      */
     public static getMinimalPrompt(): string {
         const plugins = getAllPluginMetadata();
-        const pluginList = plugins.map(p => `- \`${p.name}\`: ${p.description}`).join('\n');
-        
+        const pluginList = plugins.map((p) => `- \`${p.name}\`: ${p.description}`).join("\n");
+
         return `You are a FlowQuery assistant. Generate FlowQuery statements based on user requests.
 
 Available data loader plugins:
@@ -379,7 +374,7 @@ Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
     /**
      * Generate the FlowQuery system prompt asynchronously using functions() introspection.
      * This is the preferred method that uses FlowQuery's built-in introspection.
-     * 
+     *
      * @param additionalContext - Optional additional context to include in the prompt
      * @returns Promise resolving to the complete system prompt string
      */
@@ -387,15 +382,19 @@ Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
         // Use FlowQuery's functions() introspection to discover available loaders
         const plugins = await getAvailableLoaders();
         const pluginDocs = this.generatePluginDocumentation(plugins);
-        
+
         return this.buildSystemPrompt(pluginDocs, additionalContext);
     }
 }
 
 // Export functions for backward compatibility
-export const generateFlowQuerySystemPrompt = FlowQuerySystemPrompt.generate.bind(FlowQuerySystemPrompt);
-export const generateInterpretationPrompt = FlowQuerySystemPrompt.generateInterpretationPrompt.bind(FlowQuerySystemPrompt);
-export const getMinimalFlowQueryPrompt = FlowQuerySystemPrompt.getMinimalPrompt.bind(FlowQuerySystemPrompt);
-export const generateFlowQuerySystemPromptAsync = FlowQuerySystemPrompt.generateAsync.bind(FlowQuerySystemPrompt);
+export const generateFlowQuerySystemPrompt =
+    FlowQuerySystemPrompt.generate.bind(FlowQuerySystemPrompt);
+export const generateInterpretationPrompt =
+    FlowQuerySystemPrompt.generateInterpretationPrompt.bind(FlowQuerySystemPrompt);
+export const getMinimalFlowQueryPrompt =
+    FlowQuerySystemPrompt.getMinimalPrompt.bind(FlowQuerySystemPrompt);
+export const generateFlowQuerySystemPromptAsync =
+    FlowQuerySystemPrompt.generateAsync.bind(FlowQuerySystemPrompt);
 
 export default FlowQuerySystemPrompt;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",

package/src/parsing/expressions/expression_map.ts

@@ -5,6 +5,9 @@ class ExpressionMap {
     public get(alias: string): Expression | undefined {
         return this._map.get(alias);
     }
+    public has(alias: string): boolean {
+        return this._map.has(alias);
+    }
     public set map(expressions: Expression[]) {
         this._map.clear();
         for (const expr of expressions) {

package/src/parsing/operations/call.ts

@@ -50,7 +50,9 @@ class Call extends Projection {
                 const record: Map<string, any> = new Map();
                 if (typeof item == "object" && !Array.isArray(item)) {
                     for (const [key, value] of Object.entries(item)) {
-                        record.set(key, value);
+                        if (this._map.has(key) || !this.hasYield) {
+                            record.set(key, value);
+                        }
                     }
                 } else {
                     record.set(DEFAULT_VARIABLE_NAME, item);

package/tests/compute/runner.test.ts

@@ -15,9 +15,9 @@ class CallTestFunction extends AsyncFunction {
         this._expectedParameterCount = 0;
     }
     public async *generate(): AsyncGenerator<any> {
-        yield { result: 1 };
-        yield { result: 2 };
-        yield { result: 3 };
+        yield { result: 1, dummy: "a" };
+        yield { result: 2, dummy: "b" };
+        yield { result: 3, dummy: "c" };
     }
 }
 
@@ -616,9 +616,9 @@ test("Test call operation as last operation", async () => {
     await runner.run();
     const results = runner.results;
     expect(results.length).toBe(3);
-    expect(results[0]).toEqual({ result: 1 });
-    expect(results[1]).toEqual({ result: 2 });
-    expect(results[2]).toEqual({ result: 3 });
+    expect(results[0]).toEqual({ result: 1, dummy: "a" });
+    expect(results[1]).toEqual({ result: 2, dummy: "b" });
+    expect(results[2]).toEqual({ result: 3, dummy: "c" });
 });
 
 test("Test call operation as last operation with yield", async () => {

package/tests/parsing/parser.test.ts

@@ -1,8 +1,5 @@
 import AsyncFunction from "../../src/parsing/functions/async_function";
-import {
-    FunctionDef,
-    getRegisteredFunctionFactory,
-} from "../../src/parsing/functions/function_metadata";
+import { FunctionDef } from "../../src/parsing/functions/function_metadata";
 import Parser from "../../src/parsing/parser";
 
 // Test class for CALL operation parsing test - defined at module level for Prettier compatibility
@@ -12,7 +9,7 @@ import Parser from "../../src/parsing/parser";
     parameters: [],
     output: { description: "Yields test values", type: "any" },
 })
-class CallParserTestFunction extends AsyncFunction {
+class Test extends AsyncFunction {
     constructor() {
         super();
         this._expectedParameterCount = 0;
@@ -473,7 +470,7 @@ test("Test return -2", () => {
 
 test("Test call operation", () => {
     const parser = new Parser();
-    const ast = parser.parse("CALL callparsertestfunction() YIELD result RETURN result");
+    const ast = parser.parse("CALL test() YIELD result RETURN result");
     expect(ast.print()).toBe(
         "ASTNode\n" +
             "- Call\n" +

package/tests/tokenization/tokenizer.test.ts

@@ -1,62 +1,62 @@
-import Tokenizer from '../../src/tokenization/tokenizer';
+import Tokenizer from "../../src/tokenization/tokenizer";
 
-test('Tokenizer.tokenize() should return an array of tokens', () => {
-    const tokenizer = new Tokenizer('MATCH (n:Person) RETURN n');
+test("Tokenizer.tokenize() should return an array of tokens", () => {
+    const tokenizer = new Tokenizer("MATCH (n:Person) RETURN n");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Tokenizer.tokenize() should handle escaped quotes', () => {
+test("Tokenizer.tokenize() should handle escaped quotes", () => {
     const tokenizer = new Tokenizer('return "hello \\"world"');
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test predicate function', () => {
-    const tokenizer = new Tokenizer('RETURN sum(n in [1, 2, 3] | n where n > 1)');
+test("Test predicate function", () => {
+    const tokenizer = new Tokenizer("RETURN sum(n in [1, 2, 3] | n where n > 1)");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test f-string', () => {
+test("Test f-string", () => {
     const tokenizer = new Tokenizer('RETURN f"hello {world}"');
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test', () => {
-    const tokenizer = new Tokenizer('WITH 1 AS n RETURN n');
+test("Test", () => {
+    const tokenizer = new Tokenizer("WITH 1 AS n RETURN n");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test associative array with backtick string', () => {
-    const tokenizer = new Tokenizer('RETURN {`key`: `value`}');
+test("Test associative array with backtick string", () => {
+    const tokenizer = new Tokenizer("RETURN {`key`: `value`}");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test limit', () => {
-    const tokenizer = new Tokenizer('unwind range(1, 10) as n limit 5 return n');
+test("Test limit", () => {
+    const tokenizer = new Tokenizer("unwind range(1, 10) as n limit 5 return n");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test return -2', () => {
-    const tokenizer = new Tokenizer('return [:-2], -2');
+test("Test return -2", () => {
+    const tokenizer = new Tokenizer("return [:-2], -2");
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
 });
 
-test('Test range with function', () => {
+test("Test range with function", () => {
     const tokenizer = new Tokenizer(`
         with range(1,10) as data
         return range(0, size(data)-1) as indices
@@ -64,4 +64,4 @@ test('Test range with function', () => {
     const tokens = tokenizer.tokenize();
     expect(tokens).toBeDefined();
     expect(tokens.length).toBeGreaterThan(0);
-});
+});