flowquery 1.0.0 → 1.0.2

package/src/parsing/functions/function_metadata.ts

@@ -0,0 +1,238 @@
+/**
+ * Schema definition for function arguments and outputs.
+ * Compatible with JSON Schema for LLM consumption.
+ */
+export interface ParameterSchema {
+    /** The parameter name */
+    name: string;
+    /** Description of the parameter */
+    description: string;
+    /** JSON Schema type: string, number, boolean, object, array, null */
+    type: "string" | "number" | "boolean" | "object" | "array" | "null" | string;
+    /** Whether the parameter is required (default: true) */
+    required?: boolean;
+    /** Default value if not provided */
+    default?: any;
+    /** For arrays, the schema of items */
+    items?: Omit<ParameterSchema, 'name' | 'required' | 'default'>;
+    /** For objects, the properties schema */
+    properties?: Record<string, Omit<ParameterSchema, 'name' | 'required'>>;
+    /** Enum of allowed values */
+    enum?: any[];
+    /** Example value */
+    example?: any;
+}
+
+/**
+ * Schema definition for function output.
+ */
+export interface OutputSchema {
+    /** Description of the output */
+    description: string;
+    /** JSON Schema type */
+    type: "string" | "number" | "boolean" | "object" | "array" | "null" | string;
+    /** For arrays, the schema of items */
+    items?: Omit<OutputSchema, 'description'>;
+    /** For objects, the properties schema */
+    properties?: Record<string, Omit<ParameterSchema, 'name' | 'required'>>;
+    /** Example output value */
+    example?: any;
+}
+
+/**
+ * Metadata for a registered function, designed for LLM consumption.
+ */
+export interface FunctionMetadata {
+    /** The function name */
+    name: string;
+    /** Human-readable description of what the function does */
+    description: string;
+    /** Category for grouping functions (e.g., "aggregation", "string", "data") */
+    category?: string;
+    /** Array of parameter schemas */
+    parameters: ParameterSchema[];
+    /** Output schema */
+    output: OutputSchema;
+    /** Example usage in FlowQuery syntax */
+    examples?: string[];
+    /** Whether this is an async data provider for LOAD operations */
+    isAsyncProvider?: boolean;
+    /** Additional notes or caveats */
+    notes?: string;
+}
+
+/**
+ * Options for registering a sync function with metadata.
+ */
+export interface RegisterFunctionOptions {
+    /** Factory function that creates the Function instance */
+    factory: () => any;
+    /** Function metadata for documentation */
+    metadata: FunctionMetadata;
+}
+
+/**
+ * Options for registering an async data provider with metadata.
+ */
+export interface RegisterAsyncProviderOptions {
+    /** Async generator or function that returns data */
+    provider: (...args: any[]) => AsyncGenerator<any, void, unknown> | Promise<any>;
+    /** Function metadata for documentation */
+    metadata: FunctionMetadata;
+}
+
+/**
+ * Built-in function metadata definitions.
+ */
+export const BUILTIN_FUNCTION_METADATA: FunctionMetadata[] = [
+    {
+        name: "sum",
+        description: "Calculates the sum of numeric values across grouped rows",
+        category: "aggregation",
+        parameters: [
+            { name: "value", description: "Numeric value to sum", type: "number" }
+        ],
+        output: { description: "Sum of all values", type: "number", example: 150 },
+        examples: ["WITH [1, 2, 3] AS nums UNWIND nums AS n RETURN sum(n)"]
+    },
+    {
+        name: "avg",
+        description: "Calculates the average of numeric values across grouped rows",
+        category: "aggregation",
+        parameters: [
+            { name: "value", description: "Numeric value to average", type: "number" }
+        ],
+        output: { description: "Average of all values", type: "number", example: 50 },
+        examples: ["WITH [10, 20, 30] AS nums UNWIND nums AS n RETURN avg(n)"]
+    },
+    {
+        name: "collect",
+        description: "Collects values into an array across grouped rows",
+        category: "aggregation",
+        parameters: [
+            { name: "value", description: "Value to collect", type: "any" }
+        ],
+        output: { description: "Array of collected values", type: "array", example: [1, 2, 3] },
+        examples: ["WITH [1, 2, 3] AS nums UNWIND nums AS n RETURN collect(n)"]
+    },
+    {
+        name: "range",
+        description: "Generates an array of sequential integers",
+        category: "generator",
+        parameters: [
+            { name: "start", description: "Starting number (inclusive)", type: "number" },
+            { name: "end", description: "Ending number (inclusive)", type: "number" }
+        ],
+        output: { description: "Array of integers from start to end", type: "array", items: { type: "number" }, example: [1, 2, 3, 4, 5] },
+        examples: ["WITH range(1, 5) AS nums RETURN nums"]
+    },
+    {
+        name: "rand",
+        description: "Generates a random number between 0 and 1",
+        category: "generator",
+        parameters: [],
+        output: { description: "Random number between 0 and 1", type: "number", example: 0.7234 },
+        examples: ["WITH rand() AS r RETURN r"]
+    },
+    {
+        name: "round",
+        description: "Rounds a number to the nearest integer",
+        category: "math",
+        parameters: [
+            { name: "value", description: "Number to round", type: "number" }
+        ],
+        output: { description: "Rounded integer", type: "number", example: 4 },
+        examples: ["WITH 3.7 AS n RETURN round(n)"]
+    },
+    {
+        name: "split",
+        description: "Splits a string into an array by a delimiter",
+        category: "string",
+        parameters: [
+            { name: "text", description: "String to split", type: "string" },
+            { name: "delimiter", description: "Delimiter to split by", type: "string" }
+        ],
+        output: { description: "Array of string parts", type: "array", items: { type: "string" }, example: ["a", "b", "c"] },
+        examples: ["WITH 'a,b,c' AS s RETURN split(s, ',')"]
+    },
+    {
+        name: "join",
+        description: "Joins an array of strings with a delimiter",
+        category: "string",
+        parameters: [
+            { name: "array", description: "Array of values to join", type: "array" },
+            { name: "delimiter", description: "Delimiter to join with", type: "string" }
+        ],
+        output: { description: "Joined string", type: "string", example: "a,b,c" },
+        examples: ["WITH ['a', 'b', 'c'] AS arr RETURN join(arr, ',')"]
+    },
+    {
+        name: "replace",
+        description: "Replaces occurrences of a pattern in a string",
+        category: "string",
+        parameters: [
+            { name: "text", description: "Source string", type: "string" },
+            { name: "pattern", description: "Pattern to find", type: "string" },
+            { name: "replacement", description: "Replacement string", type: "string" }
+        ],
+        output: { description: "String with replacements", type: "string", example: "hello world" },
+        examples: ["WITH 'hello there' AS s RETURN replace(s, 'there', 'world')"]
+    },
+    {
+        name: "stringify",
+        description: "Converts a value to its JSON string representation",
+        category: "conversion",
+        parameters: [
+            { name: "value", description: "Value to stringify", type: "any" }
+        ],
+        output: { description: "JSON string", type: "string", example: "{\"a\":1}" },
+        examples: ["WITH {a: 1} AS obj RETURN stringify(obj)"]
+    },
+    {
+        name: "tojson",
+        description: "Parses a JSON string into an object",
+        category: "conversion",
+        parameters: [
+            { name: "text", description: "JSON string to parse", type: "string" }
+        ],
+        output: { description: "Parsed object or array", type: "object", example: { a: 1 } },
+        examples: ["WITH '{\"a\": 1}' AS s RETURN tojson(s)"]
+    },
+    {
+        name: "size",
+        description: "Returns the length of an array or string",
+        category: "utility",
+        parameters: [
+            { name: "value", description: "Array or string to measure", type: "array" }
+        ],
+        output: { description: "Length of the input", type: "number", example: 3 },
+        examples: ["WITH [1, 2, 3] AS arr RETURN size(arr)"]
+    },
+    {
+        name: "functions",
+        description: "Lists all registered functions with their metadata. Useful for discovering available functions and their documentation. Results include name, description, parameters, output schema, and usage examples.",
+        category: "introspection",
+        parameters: [
+            { name: "category", description: "Optional category to filter by (e.g., 'aggregation', 'string', 'math')", type: "string", required: false }
+        ],
+        output: { 
+            description: "Array of function metadata objects", 
+            type: "array",
+            items: {
+                type: "object",
+                properties: {
+                    name: { description: "Function name", type: "string" },
+                    description: { description: "What the function does", type: "string" },
+                    category: { description: "Function category", type: "string" },
+                    parameters: { description: "Array of parameter definitions", type: "array" },
+                    output: { description: "Output schema", type: "object" },
+                    examples: { description: "Usage examples", type: "array" }
+                }
+            }
+        },
+        examples: [
+            "WITH functions() AS funcs RETURN funcs",
+            "WITH functions('aggregation') AS funcs UNWIND funcs AS f RETURN f.name, f.description"
+        ]
+    }
+];

package/src/parsing/functions/functions.ts

@@ -0,0 +1,43 @@
+import Function from "./function";
+import FunctionFactory from "./function_factory";
+
+/**
+ * Built-in function that lists all registered functions with their metadata.
+ * 
+ * Can be used in FlowQuery to discover available functions:
+ * - `WITH functions() AS funcs RETURN funcs` - returns all functions
+ * - `WITH functions('aggregation') AS funcs RETURN funcs` - returns functions in a category
+ * 
+ * @example
+ * ```
+ * WITH functions() AS funcs 
+ * UNWIND funcs AS func 
+ * RETURN func.name, func.description
+ * ```
+ */
+class Functions extends Function {
+    constructor() {
+        super("functions");
+        this._expectedParameterCount = null; // 0 or 1 parameter
+    }
+
+    public value(): any {
+        const children = this.getChildren();
+        
+        if (children.length === 0) {
+            // Return all functions
+            return FunctionFactory.listFunctions();
+        } else if (children.length === 1) {
+            // Filter by category
+            const category = children[0].value();
+            if (typeof category === 'string') {
+                return FunctionFactory.listFunctions({ category });
+            }
+            throw new Error("functions() category parameter must be a string");
+        } else {
+            throw new Error("functions() takes 0 or 1 parameters");
+        }
+    }
+}
+
+export default Functions;

package/src/parsing/operations/load.ts

@@ -3,12 +3,14 @@ import CSV from "../components/csv";
 import {default as _JSON} from "../components/json";
 import Text from "../components/text";
 import Function from "../functions/function";
+import AsyncFunction from "../functions/async_function";
 import AssociativeArray from "../data_structures/associative_array";
 import Reference from "../expressions/reference";
 import Expression from "../expressions/expression";
 import Headers from "../components/headers";
 import Post from "../components/post";
 import Lookup from "../data_structures/lookup";
+import From from "../components/from";
 
 class Load extends Operation {
   private _value: any = null;
@@ -18,6 +20,29 @@ class Load extends Operation {
   public get type(): _JSON | CSV | Text {
     return this.children[0] as _JSON | CSV | Text;
   }
+
+  /**
+   * Gets the From component which contains either a URL expression or an AsyncFunction.
+   */
+  public get fromComponent(): From {
+    return this.children[1] as From;
+  }
+
+  /**
+   * Checks if the data source is an async function.
+   */
+  public get isAsyncFunction(): boolean {
+    return this.fromComponent.firstChild() instanceof AsyncFunction;
+  }
+
+  /**
+   * Gets the async function if the source is a function, otherwise null.
+   */
+  public get asyncFunction(): AsyncFunction | null {
+    const child = this.fromComponent.firstChild();
+    return child instanceof AsyncFunction ? child : null;
+  }
+
   public get from(): string {
     return this.children[1].value() as string;
   }
@@ -56,7 +81,22 @@ class Load extends Operation {
       ...(payload !== null ? {"body": JSON.stringify(payload.value())} : {})
     };
   }
-  public async load(): Promise<any> {
+
+  /**
+   * Loads data from an async function source.
+   */
+  private async loadFromFunction(): Promise<void> {
+    const asyncFunc = this.asyncFunction!;
+    for await (const item of asyncFunc.execute()) {
+      this._value = item;
+      await this.next?.run();
+    }
+  }
+
+  /**
+   * Loads data from a URL source (original behavior).
+   */
+  private async loadFromUrl(): Promise<void> {
     const result = await fetch(this.from, this.options());
     let data: any = null;
     if(this.type instanceof _JSON) {
@@ -77,11 +117,20 @@ class Load extends Operation {
       await this.next?.run();
     }
   }
+
+  public async load(): Promise<any> {
+    if (this.isAsyncFunction) {
+      await this.loadFromFunction();
+    } else {
+      await this.loadFromUrl();
+    }
+  }
   public async run(): Promise<void> {
     try {
       await this.load();
     } catch(e) {
-      throw new Error(`Failed to load data from ${this.from}. Error: ${e}`);
+      const source = this.isAsyncFunction ? this.asyncFunction?.name : this.from;
+      throw new Error(`Failed to load data from ${source}. Error: ${e}`);
     }
   }
   public value(): any {

package/src/parsing/parser.ts

@@ -4,6 +4,7 @@ import ASTNode from "./ast_node";
 import BaseParser from "./base_parser";
 import FunctionFactory from "./functions/function_factory";
 import Function from "./functions/function";
+import AsyncFunction from "./functions/async_function";
 import AssociativeArray from "./data_structures/associative_array";
 import JSONArray from "./data_structures/json_array";
 import KeyValuePair from "./data_structures/key_value_pair";
@@ -218,11 +219,19 @@ class Parser extends BaseParser {
         this.expectAndSkipWhitespaceAndComments();
         const from = new From();
         load.addChild(from);
-        const expression = this.parseExpression();
-        if(expression === null) {
-            throw new Error('Expected expression');
+        
+        // Check if the source is an async function
+        const asyncFunc = this.parseAsyncFunction();
+        if (asyncFunc !== null) {
+            from.addChild(asyncFunc);
+        } else {
+            const expression = this.parseExpression();
+            if(expression === null) {
+                throw new Error('Expected expression or async function');
+            }
+            from.addChild(expression);
         }
-        from.addChild(expression);
+        
         this.expectAndSkipWhitespaceAndComments();
         if(this.token.isHeaders()) {
             const headers = new Headers();
@@ -569,6 +578,38 @@ class Parser extends BaseParser {
         return func;
     }
 
+    /**
+     * Parses an async function call for use in LOAD operations.
+     * Only matches if the identifier is registered as an async data provider.
+     * 
+     * @returns An AsyncFunction node if a registered async function is found, otherwise null
+     */
+    private parseAsyncFunction(): AsyncFunction | null {
+        if(!this.token.isIdentifier()) {
+            return null;
+        }
+        if(this.token.value === null) {
+            return null;
+        }
+        // Only parse as async function if it's registered as an async provider
+        if(!FunctionFactory.isAsyncProvider(this.token.value)) {
+            return null;
+        }
+        if(!this.peek()?.isLeftParenthesis()) {
+            return null;
+        }
+        const asyncFunc = new AsyncFunction(this.token.value);
+        this.setNextToken(); // skip function name
+        this.setNextToken(); // skip left parenthesis
+        this.skipWhitespaceAndComments();
+        asyncFunc.parameters = Array.from(this.parseExpressions(AliasOption.NOT_ALLOWED));
+        this.skipWhitespaceAndComments();
+        if(!this.token.isRightParenthesis()) {
+            throw new Error('Expected right parenthesis');
+        }
+        this.setNextToken();
+        return asyncFunc;
+    }
     private parsePredicateFunction(): PredicateFunction | null {
         if(!this.ahead([Token.IDENTIFIER(""), Token.LEFT_PARENTHESIS, Token.IDENTIFIER(""), Token.IN])) {
             return null;