flowquery 1.0.0 → 1.0.2

package/src/index.browser.ts

@@ -7,5 +7,123 @@
  */
 
 import {default as FlowQuery} from "./compute/runner";
+import FunctionFactory, { FunctionCreator, AsyncDataProvider } from "./parsing/functions/function_factory";
+import { 
+    FunctionMetadata, 
+    ParameterSchema, 
+    OutputSchema,
+    RegisterFunctionOptions,
+    RegisterAsyncProviderOptions 
+} from "./parsing/functions/function_metadata";
+import Function from "./parsing/functions/function";
+
+/**
+ * Register a synchronous plugin function.
+ * 
+ * @param name - The function name
+ * @param factoryOrOptions - Factory function or options object with metadata
+ * 
+ * @example
+ * ```javascript
+ * // Simple registration
+ * FlowQuery.registerFunction("uppercase", () => new MyUpperCaseFunction());
+ * 
+ * // Registration with metadata for LLM consumption
+ * FlowQuery.registerFunction("uppercase", {
+ *     factory: () => new MyUpperCaseFunction(),
+ *     metadata: {
+ *         name: "uppercase",
+ *         description: "Converts a string to uppercase",
+ *         category: "string",
+ *         parameters: [{ name: "text", description: "String to convert", type: "string" }],
+ *         output: { description: "Uppercase string", type: "string" },
+ *         examples: ["WITH 'hello' AS s RETURN uppercase(s)"]
+ *     }
+ * });
+ * ```
+ */
+FlowQuery.registerFunction = function(name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions): void {
+    FunctionFactory.register(name, factoryOrOptions);
+};
+
+/**
+ * Unregister a synchronous plugin function.
+ * 
+ * @param name - The function name
+ */
+FlowQuery.unregisterFunction = function(name: string): void {
+    FunctionFactory.unregister(name);
+};
+
+/**
+ * Register an async data provider function for use in LOAD operations.
+ * 
+ * @param name - The function name
+ * @param providerOrOptions - Async provider or options object with metadata
+ * 
+ * @example
+ * ```javascript
+ * // Registration with metadata for LLM consumption
+ * FlowQuery.registerAsyncProvider("fetchUsers", {
+ *     provider: async function* (endpoint) {
+ *         const response = await fetch(endpoint);
+ *         const data = await response.json();
+ *         for (const item of data) yield item;
+ *     },
+ *     metadata: {
+ *         name: "fetchUsers",
+ *         description: "Fetches user data from an API",
+ *         category: "data",
+ *         parameters: [{ name: "endpoint", description: "API URL", type: "string" }],
+ *         output: { 
+ *             description: "User object", 
+ *             type: "object",
+ *             properties: {
+ *                 id: { description: "User ID", type: "number" },
+ *                 name: { description: "User name", type: "string" }
+ *             }
+ *         },
+ *         examples: ["LOAD JSON FROM fetchUsers('https://api.example.com/users') AS user"]
+ *     }
+ * });
+ * ```
+ */
+FlowQuery.registerAsyncProvider = function(name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions): void {
+    FunctionFactory.registerAsyncProvider(name, providerOrOptions);
+};
+
+/**
+ * Unregister an async data provider function.
+ * 
+ * @param name - The function name
+ */
+FlowQuery.unregisterAsyncProvider = function(name: string): void {
+    FunctionFactory.unregisterAsyncProvider(name);
+};
+
+/**
+ * List all registered functions with their metadata.
+ * 
+ * @param options - Optional filter options
+ * @returns Array of function metadata
+ */
+FlowQuery.listFunctions = function(options?: { category?: string; asyncOnly?: boolean; syncOnly?: boolean }): FunctionMetadata[] {
+    return FunctionFactory.listFunctions(options);
+};
+
+/**
+ * Get metadata for a specific function.
+ * 
+ * @param name - The function name
+ * @returns Function metadata or undefined
+ */
+FlowQuery.getFunctionMetadata = function(name: string): FunctionMetadata | undefined {
+    return FunctionFactory.getMetadata(name);
+};
+
+/**
+ * Base Function class for creating custom plugin functions.
+ */
+FlowQuery.Function = Function;
 
 export default FlowQuery;

package/src/index.node.ts

@@ -0,0 +1,141 @@
+/**
+ * FlowQuery - A declarative query language for data processing pipelines.
+ * 
+ * This is the main entry point for the FlowQuery Node.js library usage.
+ * 
+ * @packageDocumentation
+ */
+
+import {default as FlowQuery} from "./compute/runner";
+import FunctionFactory, { FunctionCreator, AsyncDataProvider } from "./parsing/functions/function_factory";
+import { 
+    FunctionMetadata, 
+    ParameterSchema, 
+    OutputSchema,
+    RegisterFunctionOptions,
+    RegisterAsyncProviderOptions 
+} from "./parsing/functions/function_metadata";
+import Function from "./parsing/functions/function";
+
+/**
+ * Register a synchronous plugin function.
+ * 
+ * @param name - The function name
+ * @param factoryOrOptions - Factory function or options object with metadata
+ * 
+ * @example
+ * ```javascript
+ * // Simple registration
+ * FlowQuery.registerFunction("uppercase", () => new MyUpperCaseFunction());
+ * 
+ * // Registration with metadata for LLM consumption
+ * FlowQuery.registerFunction("uppercase", {
+ *     factory: () => new MyUpperCaseFunction(),
+ *     metadata: {
+ *         name: "uppercase",
+ *         description: "Converts a string to uppercase",
+ *         category: "string",
+ *         parameters: [{ name: "text", description: "String to convert", type: "string" }],
+ *         output: { description: "Uppercase string", type: "string" },
+ *         examples: ["WITH 'hello' AS s RETURN uppercase(s)"]
+ *     }
+ * });
+ * ```
+ */
+FlowQuery.registerFunction = function(name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions): void {
+    FunctionFactory.register(name, factoryOrOptions);
+};
+
+/**
+ * Unregister a synchronous plugin function.
+ * 
+ * @param name - The function name
+ */
+FlowQuery.unregisterFunction = function(name: string): void {
+    FunctionFactory.unregister(name);
+};
+
+/**
+ * Register an async data provider function for use in LOAD operations.
+ * 
+ * @param name - The function name
+ * @param providerOrOptions - Async provider or options object with metadata
+ * 
+ * @example
+ * ```javascript
+ * // Registration with metadata for LLM consumption
+ * FlowQuery.registerAsyncProvider("fetchUsers", {
+ *     provider: async function* (endpoint) {
+ *         const response = await fetch(endpoint);
+ *         const data = await response.json();
+ *         for (const item of data) yield item;
+ *     },
+ *     metadata: {
+ *         name: "fetchUsers",
+ *         description: "Fetches user data from an API",
+ *         category: "data",
+ *         parameters: [{ name: "endpoint", description: "API URL", type: "string" }],
+ *         output: { 
+ *             description: "User object", 
+ *             type: "object",
+ *             properties: {
+ *                 id: { description: "User ID", type: "number" },
+ *                 name: { description: "User name", type: "string" }
+ *             }
+ *         },
+ *         examples: ["LOAD JSON FROM fetchUsers('https://api.example.com/users') AS user"]
+ *     }
+ * });
+ * ```
+ */
+FlowQuery.registerAsyncProvider = function(name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions): void {
+    FunctionFactory.registerAsyncProvider(name, providerOrOptions);
+};
+
+/**
+ * Unregister an async data provider function.
+ * 
+ * @param name - The function name
+ */
+FlowQuery.unregisterAsyncProvider = function(name: string): void {
+    FunctionFactory.unregisterAsyncProvider(name);
+};
+
+/**
+ * List all registered functions with their metadata.
+ * 
+ * @param options - Optional filter options
+ * @returns Array of function metadata
+ */
+FlowQuery.listFunctions = function(options?: { category?: string; asyncOnly?: boolean; syncOnly?: boolean }): FunctionMetadata[] {
+    return FunctionFactory.listFunctions(options);
+};
+
+/**
+ * Get metadata for a specific function.
+ * 
+ * @param name - The function name
+ * @returns Function metadata or undefined
+ */
+FlowQuery.getFunctionMetadata = function(name: string): FunctionMetadata | undefined {
+    return FunctionFactory.getMetadata(name);
+};
+
+/**
+ * Base Function class for creating custom plugin functions.
+ */
+FlowQuery.Function = Function;
+
+export default FlowQuery;
+export { 
+    FlowQuery, 
+    Function, 
+    FunctionFactory, 
+    FunctionCreator, 
+    AsyncDataProvider,
+    FunctionMetadata,
+    ParameterSchema,
+    OutputSchema,
+    RegisterFunctionOptions,
+    RegisterAsyncProviderOptions
+};

package/src/parsing/functions/async_function.ts

@@ -0,0 +1,95 @@
+import ASTNode from "../ast_node";
+import FunctionFactory from "./function_factory";
+
+/**
+ * Represents an async data provider function call for use in LOAD operations.
+ * 
+ * This class holds the function name and arguments, and provides async iteration
+ * over the results from a registered async data provider.
+ * 
+ * @example
+ * ```typescript
+ * // Used in: LOAD JSON FROM myDataSource('arg1', 'arg2') AS data
+ * const asyncFunc = new AsyncFunction("myDataSource");
+ * asyncFunc.parameters = [arg1Node, arg2Node];
+ * for await (const item of asyncFunc.execute()) {
+ *     console.log(item);
+ * }
+ * ```
+ */
+class AsyncFunction extends ASTNode {
+    private _name: string;
+
+    /**
+     * Creates a new AsyncFunction with the given name.
+     * 
+     * @param name - The function name (must be registered as an async provider)
+     */
+    constructor(name: string) {
+        super();
+        this._name = name;
+    }
+
+    /**
+     * Gets the function name.
+     */
+    public get name(): string {
+        return this._name;
+    }
+
+    /**
+     * Sets the function parameters.
+     * 
+     * @param nodes - Array of AST nodes representing the function arguments
+     */
+    public set parameters(nodes: ASTNode[]) {
+        this.children = nodes;
+    }
+
+    /**
+     * Evaluates all parameters and returns their values.
+     * 
+     * @returns Array of parameter values
+     */
+    private getArguments(): any[] {
+        return this.children.map(child => child.value());
+    }
+
+    /**
+     * Executes the async data provider function and yields results.
+     * 
+     * @yields Data items from the async provider
+     * @throws {Error} If the function is not registered as an async provider
+     */
+    public async *execute(): AsyncGenerator<any, void, unknown> {
+        const provider = FunctionFactory.getAsyncProvider(this._name);
+        if (!provider) {
+            throw new Error(`Async data provider '${this._name}' is not registered`);
+        }
+
+        const args = this.getArguments();
+        const result = provider(...args);
+
+        // Check if result is an async generator or a promise
+        if (result && typeof (result as AsyncGenerator).next === 'function') {
+            // It's an async generator
+            yield* result as AsyncGenerator<any, void, unknown>;
+        } else {
+            // It's a promise - await and yield the result
+            const data = await result;
+            if (Array.isArray(data)) {
+                for (const item of data) {
+                    yield item;
+                }
+            } else {
+                yield data;
+            }
+        }
+    }
+
+    public toString(): string {
+        return `AsyncFunction (${this._name})`;
+    }
+}
+
+export default AsyncFunction;

package/src/parsing/functions/function_factory.ts

@@ -10,7 +10,25 @@ import ToJson from "./to_json";
 import Replace from "./replace";
 import Stringify from "./stringify";
 import Size from "./size";
+import Functions from "./functions";
 import Function from "./function";
+import { 
+    FunctionMetadata, 
+    RegisterFunctionOptions, 
+    RegisterAsyncProviderOptions,
+    BUILTIN_FUNCTION_METADATA 
+} from "./function_metadata";
+
+/**
+ * Type for synchronous function factories.
+ */
+export type FunctionCreator = () => Function;
+
+/**
+ * Type for async data provider functions used in LOAD operations.
+ * These functions can yield data asynchronously.
+ */
+export type AsyncDataProvider = (...args: any[]) => AsyncGenerator<any, void, unknown> | Promise<any>;
 
 /**
  * Factory for creating function instances by name.
@@ -25,6 +43,208 @@ import Function from "./function";
  * ```
  */
 class FunctionFactory {
+    /**
+     * Registry for plugin functions (synchronous).
+     */
+    private static plugins: Map<string, FunctionCreator> = new Map();
+
+    /**
+     * Registry for async data provider functions used in LOAD operations.
+     */
+    private static asyncProviders: Map<string, AsyncDataProvider> = new Map();
+
+    /**
+     * Registry for function metadata (both sync and async).
+     */
+    private static metadata: Map<string, FunctionMetadata> = new Map();
+
+    /**
+     * Registers a synchronous plugin function.
+     * 
+     * @param name - The function name (will be lowercased)
+     * @param factoryOrOptions - Factory function or options object with metadata
+     */
+    public static register(name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions): void {
+        const lowerName = name.toLowerCase();
+        if (typeof factoryOrOptions === 'function') {
+            FunctionFactory.plugins.set(lowerName, factoryOrOptions);
+        } else {
+            FunctionFactory.plugins.set(lowerName, factoryOrOptions.factory);
+            FunctionFactory.metadata.set(lowerName, {
+                ...factoryOrOptions.metadata,
+                name: lowerName,
+                isAsyncProvider: false
+            });
+        }
+    }
+
+    /**
+     * Unregisters a synchronous plugin function.
+     * 
+     * @param name - The function name to unregister
+     */
+    public static unregister(name: string): void {
+        const lowerName = name.toLowerCase();
+        FunctionFactory.plugins.delete(lowerName);
+        FunctionFactory.metadata.delete(lowerName);
+    }
+
+    /**
+     * Registers an async data provider function for use in LOAD operations.
+     * 
+     * @param name - The function name (will be lowercased)
+     * @param providerOrOptions - Async provider or options object with metadata
+     * 
+     * @example
+     * ```typescript
+     * // Register with metadata for LLM consumption
+     * FunctionFactory.registerAsyncProvider("fetchUsers", {
+     *     provider: async function* (endpoint: string) {
+     *         const response = await fetch(endpoint);
+     *         const data = await response.json();
+     *         for (const item of data) {
+     *             yield item;
+     *         }
+     *     },
+     *     metadata: {
+     *         name: "fetchUsers",
+     *         description: "Fetches user data from an API endpoint",
+     *         parameters: [
+     *             { name: "endpoint", description: "API endpoint URL", type: "string" }
+     *         ],
+     *         output: {
+     *             description: "User objects",
+     *             type: "object",
+     *             properties: {
+     *                 id: { description: "User ID", type: "number" },
+     *                 name: { description: "User name", type: "string" }
+     *             }
+     *         },
+     *         examples: ["LOAD JSON FROM fetchUsers('https://api.example.com/users') AS user"]
+     *     }
+     * });
+     * ```
+     */
+    public static registerAsyncProvider(name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions): void {
+        const lowerName = name.toLowerCase();
+        if (typeof providerOrOptions === 'function') {
+            FunctionFactory.asyncProviders.set(lowerName, providerOrOptions);
+        } else {
+            FunctionFactory.asyncProviders.set(lowerName, providerOrOptions.provider);
+            FunctionFactory.metadata.set(lowerName, {
+                ...providerOrOptions.metadata,
+                name: lowerName,
+                isAsyncProvider: true
+            });
+        }
+    }
+
+    /**
+     * Unregisters an async data provider function.
+     * 
+     * @param name - The function name to unregister
+     */
+    public static unregisterAsyncProvider(name: string): void {
+        const lowerName = name.toLowerCase();
+        FunctionFactory.asyncProviders.delete(lowerName);
+        FunctionFactory.metadata.delete(lowerName);
+    }
+
+    /**
+     * Gets an async data provider by name.
+     * 
+     * @param name - The function name (case-insensitive)
+     * @returns The async data provider, or undefined if not found
+     */
+    public static getAsyncProvider(name: string): AsyncDataProvider | undefined {
+        return FunctionFactory.asyncProviders.get(name.toLowerCase());
+    }
+
+    /**
+     * Checks if a function name is registered as an async data provider.
+     * 
+     * @param name - The function name (case-insensitive)
+     * @returns True if the function is an async data provider
+     */
+    public static isAsyncProvider(name: string): boolean {
+        return FunctionFactory.asyncProviders.has(name.toLowerCase());
+    }
+
+    /**
+     * Gets metadata for a specific function.
+     * 
+     * @param name - The function name (case-insensitive)
+     * @returns The function metadata, or undefined if not found
+     */
+    public static getMetadata(name: string): FunctionMetadata | undefined {
+        const lowerName = name.toLowerCase();
+        // Check plugin metadata first
+        if (FunctionFactory.metadata.has(lowerName)) {
+            return FunctionFactory.metadata.get(lowerName);
+        }
+        // Fall back to built-in metadata
+        return BUILTIN_FUNCTION_METADATA.find(m => m.name === lowerName);
+    }
+
+    /**
+     * Lists all registered functions with their metadata.
+     * Includes both built-in and plugin functions.
+     * 
+     * @param options - Optional filter options
+     * @returns Array of function metadata
+     */
+    public static listFunctions(options?: { 
+        category?: string; 
+        includeBuiltins?: boolean;
+        asyncOnly?: boolean;
+        syncOnly?: boolean;
+    }): FunctionMetadata[] {
+        const result: FunctionMetadata[] = [];
+        const includeBuiltins = options?.includeBuiltins !== false;
+        
+        // Add built-in functions
+        if (includeBuiltins) {
+            for (const meta of BUILTIN_FUNCTION_METADATA) {
+                if (options?.category && meta.category !== options.category) continue;
+                if (options?.asyncOnly) continue; // Built-ins are sync
+                result.push(meta);
+            }
+        }
+        
+        // Add plugin functions
+        for (const [name, meta] of FunctionFactory.metadata) {
+            if (options?.category && meta.category !== options.category) continue;
+            if (options?.asyncOnly && !meta.isAsyncProvider) continue;
+            if (options?.syncOnly && meta.isAsyncProvider) continue;
+            result.push(meta);
+        }
+        
+        return result;
+    }
+
+    /**
+     * Lists all registered function names.
+     * 
+     * @returns Array of function names
+     */
+    public static listFunctionNames(): string[] {
+        const builtinNames = BUILTIN_FUNCTION_METADATA.map(m => m.name);
+        const pluginNames = Array.from(FunctionFactory.plugins.keys());
+        const asyncNames = Array.from(FunctionFactory.asyncProviders.keys());
+        return [...new Set([...builtinNames, ...pluginNames, ...asyncNames])];
+    }
+
+    /**
+     * Gets all function metadata as a JSON-serializable object for LLM consumption.
+     * 
+     * @returns Object with functions grouped by category
+     */
+    public static toJSON(): { functions: FunctionMetadata[]; categories: string[] } {
+        const functions = FunctionFactory.listFunctions();
+        const categories = [...new Set(functions.map(f => f.category).filter(Boolean))] as string[];
+        return { functions, categories };
+    }
+
     /**
      * Creates a function instance by name.
      * 
@@ -32,7 +252,14 @@ class FunctionFactory {
      * @returns A Function instance of the appropriate type
      */
     public static create(name: string): Function {
-        switch (name.toLowerCase()) {
+        const lowerName = name.toLowerCase();
+        
+        // Check plugin registry first (allows overriding built-ins)
+        if (FunctionFactory.plugins.has(lowerName)) {
+            return FunctionFactory.plugins.get(lowerName)!();
+        }
+
+        switch (lowerName) {
             case "sum":
                 return new Sum();
             case "collect":
@@ -57,6 +284,8 @@ class FunctionFactory {
                 return new Stringify();
             case "size":
                 return new Size();
+            case "functions":
+                return new Functions();
             default:
                 return new Function(name);
         }