flowquery 1.0.7 → 1.0.9

package/misc/apps/RAG/src/plugins/loaders/Table.ts

@@ -0,0 +1,271 @@
+/**
+ * Table loader plugin - transforms tabular data into Adaptive Card format.
+ * 
+ * Adaptive Cards are platform-agnostic UI snippets that can be rendered in
+ * Microsoft Teams, Outlook, Windows, and other applications.
+ * 
+ * Usage in FlowQuery:
+ *   // First collect data from an async provider, then pass to table:
+ *   LOAD JSON FROM mockUsers(5) AS u
+ *   WITH collect(u) AS users
+ *   LOAD JSON FROM table(users, 'Users') AS card
+ *   RETURN card
+ * 
+ * Note: Async providers cannot be nested as function arguments.
+ */
+
+import { FunctionDef } from 'flowquery/extensibility';
+
+/**
+ * Interface for Adaptive Card structure
+ */
+interface AdaptiveCard {
+    type: 'AdaptiveCard';
+    $schema: string;
+    version: string;
+    body: AdaptiveCardElement[];
+}
+
+interface AdaptiveCardElement {
+    type: string;
+    [key: string]: any;
+}
+
+interface TableCell {
+    type: 'TableCell';
+    items: AdaptiveCardElement[];
+}
+
+interface TableRow {
+    type: 'TableRow';
+    cells: TableCell[];
+}
+
+/**
+ * Table loader - transforms tabular data into an Adaptive Card table format.
+ */
+@FunctionDef({
+    description: 'Transforms tabular data into an Adaptive Card JSON format with a table layout',
+    category: 'async',
+    parameters: [
+        {
+            name: 'data',
+            description: 'Array of objects or async generator to display as a table',
+            type: 'array',
+            required: true
+        },
+        {
+            name: 'title',
+            description: 'Optional title for the card',
+            type: 'string',
+            required: false,
+            default: 'Data Table'
+        },
+        {
+            name: 'columns',
+            description: 'Optional array of column names to include (defaults to all columns from first row)',
+            type: 'array',
+            required: false
+        },
+        {
+            name: 'maxRows',
+            description: 'Maximum number of rows to display',
+            type: 'number',
+            required: false,
+            default: 100
+        }
+    ],
+    output: {
+        description: 'Adaptive Card JSON object',
+        type: 'object',
+        properties: {
+            type: { description: 'Always "AdaptiveCard"', type: 'string' },
+            $schema: { description: 'Adaptive Card schema URL', type: 'string' },
+            version: { description: 'Adaptive Card version', type: 'string' },
+            body: { description: 'Card body elements including table', type: 'array' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM mockUsers(5) AS u WITH collect(u) AS users LOAD JSON FROM table(users, 'User List') AS card RETURN card",
+        "LOAD JSON FROM mockProducts(10) AS p WITH collect(p) AS products LOAD JSON FROM table(products, 'Products', ['name', 'price', 'category']) AS card RETURN card"
+    ]
+})
+export class TableLoader {
+    /**
+     * Transforms data into an Adaptive Card with table layout.
+     * 
+     * @param data - Array or async iterable of objects
+     * @param title - Card title
+     * @param columns - Optional column names to include
+     * @param maxRows - Maximum rows to include
+     */
+    async *fetch(
+        data: any[] | AsyncIterable<any>,
+        title: string = 'Data Table',
+        columns?: string[],
+        maxRows: number = 100
+    ): AsyncGenerator<AdaptiveCard, void, unknown> {
+        // Collect data from array or async iterable
+        const rows: any[] = [];
+        
+        if (Symbol.asyncIterator in Object(data)) {
+            for await (const item of data as AsyncIterable<any>) {
+                rows.push(item);
+                if (rows.length >= maxRows) break;
+            }
+        } else if (Array.isArray(data)) {
+            rows.push(...data.slice(0, maxRows));
+        } else {
+            // Single object
+            rows.push(data);
+        }
+
+        if (rows.length === 0) {
+            yield this.createEmptyCard(title);
+            return;
+        }
+
+        // Determine columns from first row if not specified
+        const columnNames = columns || Object.keys(rows[0]);
+
+        yield this.createTableCard(title, columnNames, rows);
+    }
+
+    /**
+     * Creates an Adaptive Card with a table displaying the data.
+     * Uses ColumnSet/Column for better compatibility across renderers.
+     */
+    private createTableCard(title: string, columnNames: string[], rows: any[]): AdaptiveCard {
+        const card: AdaptiveCard = {
+            type: 'AdaptiveCard',
+            $schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
+            version: '1.3',
+            body: []
+        };
+
+        // Add title
+        card.body.push({
+            type: 'TextBlock',
+            text: title,
+            weight: 'Bolder',
+            size: 'Large',
+            wrap: true
+        });
+
+        // Add separator
+        card.body.push({
+            type: 'TextBlock',
+            text: ' ',
+            separator: true
+        });
+
+        // Create header row using ColumnSet
+        const headerColumnSet: AdaptiveCardElement = {
+            type: 'ColumnSet',
+            columns: columnNames.map(col => ({
+                type: 'Column',
+                width: 'stretch',
+                items: [{
+                    type: 'TextBlock',
+                    text: this.formatColumnName(col),
+                    weight: 'Bolder',
+                    wrap: true
+                }]
+            })),
+            style: 'accent'
+        };
+        card.body.push(headerColumnSet);
+
+        // Add data rows using ColumnSets
+        for (const row of rows) {
+            const dataColumnSet: AdaptiveCardElement = {
+                type: 'ColumnSet',
+                columns: columnNames.map(col => ({
+                    type: 'Column',
+                    width: 'stretch',
+                    items: [{
+                        type: 'TextBlock',
+                        text: this.formatCellValue(row[col]),
+                        wrap: true
+                    }]
+                })),
+                separator: true
+            };
+            card.body.push(dataColumnSet);
+        }
+
+        // Add row count footer
+        card.body.push({
+            type: 'TextBlock',
+            text: `Showing ${rows.length} row${rows.length !== 1 ? 's' : ''}`,
+            size: 'Small',
+            isSubtle: true,
+            horizontalAlignment: 'Right',
+            separator: true
+        });
+
+        return card;
+    }
+
+    /**
+     * Creates an empty card when no data is available.
+     */
+    private createEmptyCard(title: string): AdaptiveCard {
+        return {
+            type: 'AdaptiveCard',
+            $schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
+            version: '1.3',
+            body: [
+                {
+                    type: 'TextBlock',
+                    text: title,
+                    weight: 'Bolder',
+                    size: 'Large',
+                    wrap: true
+                },
+                {
+                    type: 'TextBlock',
+                    text: 'No data available',
+                    isSubtle: true,
+                    wrap: true
+                }
+            ]
+        };
+    }
+
+    /**
+     * Formats a column name for display (converts camelCase/snake_case to Title Case).
+     */
+    private formatColumnName(name: string): string {
+        return name
+            .replace(/([A-Z])/g, ' $1')  // camelCase
+            .replace(/_/g, ' ')           // snake_case
+            .replace(/^\w/, c => c.toUpperCase())
+            .trim();
+    }
+
+    /**
+     * Formats a cell value for display in the table.
+     */
+    private formatCellValue(value: any): string {
+        if (value === null || value === undefined) {
+            return '-';
+        }
+        if (typeof value === 'boolean') {
+            return value ? '✓' : '✗';
+        }
+        if (typeof value === 'number') {
+            // Format numbers nicely
+            if (Number.isInteger(value)) {
+                return value.toString();
+            }
+            return value.toFixed(2);
+        }
+        if (typeof value === 'object') {
+            return JSON.stringify(value);
+        }
+        return String(value);
+    }
+}
+
+export { TableLoader as default };

package/misc/apps/RAG/src/prompts/FlowQuerySystemPrompt.ts

@@ -33,6 +33,18 @@ FlowQuery is a declarative query language for data processing pipelines. It uses
    LOAD JSON FROM 'https://api.example.com/data' AS item
    LOAD JSON FROM myFunction(arg1, arg2) AS item
    \`\`\`
+   
+   **IMPORTANT**: Async data providers (functions used after LOAD JSON FROM) cannot be nested inside other function calls. If you need to pass data from one async provider to another, first load the data into a variable using collect(), then pass that variable:
+   \`\`\`
+   // WRONG - async providers cannot be nested:
+   // LOAD JSON FROM table(mockProducts(5), 'Products') AS card
+   
+   // CORRECT - collect data first, then pass to next provider:
+   LOAD JSON FROM mockProducts(5) AS p
+   WITH collect(p) AS products
+   LOAD JSON FROM table(products, 'Products') AS card
+   RETURN card
+   \`\`\`
 
 3. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
    \`\`\`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",

package/src/parsing/functions/function_factory.ts

@@ -9,6 +9,7 @@ import "./rand";
 import "./round";
 import "./split";
 import "./join";
+import "./keys";
 import "./to_json";
 import "./replace";
 import "./stringify";

package/src/parsing/functions/function_metadata.ts

@@ -66,26 +66,73 @@ export interface FunctionMetadata {
     notes?: string;
 }
 
-/**
- * Registry for function metadata collected via decorators.
- */
-const functionMetadataRegistry: Map<string, FunctionMetadata> = new Map();
-
-/**
- * Registry for function factories collected via decorators.
- * Allows @FunctionDef to automatically register functions for instantiation.
- */
-const functionFactoryRegistry: Map<string, () => any> = new Map();
-
 /**
  * Type for async data provider functions used in LOAD operations.
  */
 export type AsyncDataProvider = (...args: any[]) => AsyncGenerator<any, void, unknown> | Promise<any>;
 
 /**
- * Registry for async data providers collected via decorators.
+ * Centralized registry for function metadata, factories, and async providers.
+ * Encapsulates all registration logic for the @FunctionDef decorator.
  */
-const asyncProviderRegistry: Map<string, AsyncDataProvider> = new Map();
+class FunctionRegistry {
+    private static metadata: Map<string, FunctionMetadata> = new Map<string, FunctionMetadata>();
+    private static factories: Map<string, () => any> = new Map<string, () => any>();
+    private static asyncProviders: Map<string, AsyncDataProvider> = new Map<string, AsyncDataProvider>();
+
+    /** Derives a camelCase display name from a class name, removing 'Loader' suffix. */
+    private static deriveDisplayName(className: string): string {
+        const baseName: string = className.endsWith('Loader') ? className.slice(0, -6) : className;
+        return baseName.charAt(0).toLowerCase() + baseName.slice(1);
+    }
+
+    /** Registers an async data provider class. */
+    static registerAsync<T extends new (...args: any[]) => any>(constructor: T, options: FunctionDefOptions): void {
+        const displayName: string = this.deriveDisplayName(constructor.name);
+        const registryKey: string = displayName.toLowerCase();
+
+        this.metadata.set(registryKey, { name: displayName, ...options });
+        this.asyncProviders.set(registryKey, (...args: any[]) => new constructor().fetch(...args));
+    }
+
+    /** Registers a regular function class. */
+    static registerFunction<T extends new (...args: any[]) => any>(constructor: T, options: FunctionDefOptions): void {
+        const instance: any = new constructor();
+        const baseName: string = (instance.name?.toLowerCase() || constructor.name.toLowerCase());
+        const displayName: string = baseName.includes(':') ? baseName.split(':')[0] : baseName;
+        const registryKey: string = options.category ? `${displayName}:${options.category}` : displayName;
+
+        this.metadata.set(registryKey, { name: displayName, ...options });
+        
+        if (options.category !== 'predicate') {
+            this.factories.set(displayName, () => new constructor());
+        }
+        this.factories.set(registryKey, () => new constructor());
+    }
+
+    static getAllMetadata(): FunctionMetadata[] {
+        return Array.from(this.metadata.values());
+    }
+
+    static getMetadata(name: string, category?: string): FunctionMetadata | undefined {
+        const lowerName: string = name.toLowerCase();
+        if (category) return this.metadata.get(`${lowerName}:${category}`);
+        for (const meta of this.metadata.values()) {
+            if (meta.name === lowerName) return meta;
+        }
+        return undefined;
+    }
+
+    static getFactory(name: string, category?: string): (() => any) | undefined {
+        const lowerName: string = name.toLowerCase();
+        if (category) return this.factories.get(`${lowerName}:${category}`);
+        return this.factories.get(lowerName);
+    }
+
+    static getAsyncProvider(name: string): AsyncDataProvider | undefined {
+        return this.asyncProviders.get(name.toLowerCase());
+    }
+}
 
 /**
  * Decorator options - metadata without the name (derived from class).
@@ -131,127 +178,39 @@ export type FunctionDefOptions = Omit<FunctionMetadata, 'name'>;
  */
 export function FunctionDef(options: FunctionDefOptions) {
     return function <T extends new (...args: any[]) => any>(constructor: T): T {
-        // Handle async providers differently
         if (options.category === 'async') {
-            // Derive the function name from the class name
-            // Remove 'Loader' suffix if present and convert to lowercase for registry
-            let baseName = constructor.name;
-            if (baseName.endsWith('Loader')) {
-                baseName = baseName.slice(0, -6);
-            }
-            // Keep display name in camelCase, but use lowercase for registry keys
-            const displayName = baseName.charAt(0).toLowerCase() + baseName.slice(1);
-            const registryKey = displayName.toLowerCase();
-            
-            // Register metadata with display name
-            const metadata: FunctionMetadata = {
-                name: displayName,
-                ...options
-            };
-            functionMetadataRegistry.set(registryKey, metadata);
-            
-            // Register the async provider (wraps the class's fetch method)
-            asyncProviderRegistry.set(registryKey, (...args: any[]) => new constructor().fetch(...args));
-            
-            return constructor;
-        }
-        
-        // Regular function handling
-        // Create an instance to get the function name from super() call
-        const instance = new constructor();
-        const baseName = instance.name?.toLowerCase() || constructor.name.toLowerCase();
-        
-        // Use category-qualified key to avoid collisions (e.g., sum vs sum:predicate)
-        // but store the display name without the qualifier
-        const displayName = baseName.includes(':') ? baseName.split(':')[0] : baseName;
-        const registryKey = options.category ? `${displayName}:${options.category}` : displayName;
-        
-        // Register metadata with display name but category-qualified key
-        const metadata: FunctionMetadata = {
-            name: displayName,
-            ...options
-        };
-        functionMetadataRegistry.set(registryKey, metadata);
-        
-        // Register factory function for automatic instantiation
-        // Only register to the simple name if no collision exists (predicate functions use qualified keys)
-        if (options.category !== 'predicate') {
-            functionFactoryRegistry.set(displayName, () => new constructor());
+            FunctionRegistry.registerAsync(constructor, options);
+        } else {
+            FunctionRegistry.registerFunction(constructor, options);
         }
-        functionFactoryRegistry.set(registryKey, () => new constructor());
-        
         return constructor;
     };
 }
 
 /**
  * Gets all registered function metadata from decorators.
- * 
- * @returns Array of function metadata
  */
 export function getRegisteredFunctionMetadata(): FunctionMetadata[] {
-    return Array.from(functionMetadataRegistry.values());
+    return FunctionRegistry.getAllMetadata();
 }
 
 /**
  * Gets a registered function factory by name.
- * Used by FunctionFactory to instantiate decorator-registered functions.
- * 
- * @param name - Function name (case-insensitive)
- * @param category - Optional category to disambiguate (e.g., 'predicate')
- * @returns Factory function or undefined
  */
 export function getRegisteredFunctionFactory(name: string, category?: string): (() => any) | undefined {
-    const lowerName = name.toLowerCase();
-    
-    // If category specified, look for exact match
-    if (category) {
-        return functionFactoryRegistry.get(`${lowerName}:${category}`);
-    }
-    
-    // Try direct match first
-    if (functionFactoryRegistry.has(lowerName)) {
-        return functionFactoryRegistry.get(lowerName);
-    }
-    
-    return undefined;
+    return FunctionRegistry.getFactory(name, category);
 }
 
 /**
  * Gets metadata for a specific function by name.
- * If multiple functions share the same name (e.g., aggregate vs predicate),
- * optionally specify the category to get the specific one.
- * 
- * @param name - Function name (case-insensitive)
- * @param category - Optional category to disambiguate
- * @returns Function metadata or undefined
  */
 export function getFunctionMetadata(name: string, category?: string): FunctionMetadata | undefined {
-    const lowerName = name.toLowerCase();
-    
-    // If category specified, look for exact match
-    if (category) {
-        return functionMetadataRegistry.get(`${lowerName}:${category}`);
-    }
-    
-    // Otherwise, first try direct match (for functions without category conflicts)
-    // Then search for any function with matching name
-    for (const [key, meta] of functionMetadataRegistry) {
-        if (meta.name === lowerName) {
-            return meta;
-        }
-    }
-    
-    return undefined;
+    return FunctionRegistry.getMetadata(name, category);
 }
 
 /**
  * Gets a registered async data provider by name.
- * Used by FunctionFactory to get decorator-registered async providers.
- * 
- * @param name - Function name (case-insensitive)
- * @returns Async data provider or undefined
  */
 export function getRegisteredAsyncProvider(name: string): AsyncDataProvider | undefined {
-    return asyncProviderRegistry.get(name.toLowerCase());
+    return FunctionRegistry.getAsyncProvider(name);
 }

package/src/parsing/functions/keys.ts

@@ -0,0 +1,31 @@
+import Function from "./function";
+import { FunctionDef } from "./function_metadata";
+
+@FunctionDef({
+    description: "Returns the keys of an object (associative array) as an array",
+    category: "scalar",
+    parameters: [
+        { name: "object", description: "Object to extract keys from", type: "object" }
+    ],
+    output: { description: "Array of keys", type: "array", example: "['name', 'age']" },
+    examples: ["WITH { name: 'Alice', age: 30 } AS obj RETURN keys(obj)"]
+})
+class Keys extends Function {
+    constructor() {
+        super("keys");
+        this._expectedParameterCount = 1;
+    }
+
+    public value(): any {
+        const obj = this.getChildren()[0].value();
+        if (obj === null || obj === undefined) {
+            return [];
+        }
+        if (typeof obj !== "object" || Array.isArray(obj)) {
+            throw new Error("keys() expects an object, not an array or primitive");
+        }
+        return Object.keys(obj);
+    }
+}
+
+export default Keys;

package/tests/compute/runner.test.ts

@@ -495,4 +495,12 @@ test('Test range with size', async () => {
     const results = runner.results;
     expect(results.length).toBe(1);
     expect(results[0]).toEqual({'indices': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]});
+});
+
+test('Test keys function', async () => {
+    const runner = new Runner('RETURN keys({name: "Alice", age: 30}) as keys');
+    await runner.run();
+    const results = runner.results;
+    expect(results.length).toBe(1);
+    expect(results[0]).toEqual({'keys': ['name', 'age']});
 });

package/tests/extensibility.test.ts

@@ -560,4 +560,42 @@ describe("Plugin Functions Integration with FlowQuery", () => {
         expect(runner.results[0]).toEqual({ id: 1, name: "Alice" });
         expect(runner.results[1]).toEqual({ id: 2, name: "Bob" });
     });
+
+    test("Custom function can be retrieved via functions() in a FlowQuery statement", async () => {
+        @FunctionDef({
+            description: "A unique test function for introspection",
+            category: "scalar",
+            parameters: [{ name: "x", description: "Input value", type: "number" }],
+            output: { description: "Output value", type: "number" }
+        })
+        class IntrospectTestFunc extends Function {
+            constructor() {
+                super("introspectTestFunc");
+                this._expectedParameterCount = 1;
+            }
+            
+            public value(): number {
+                return this.getChildren()[0].value() + 42;
+            }
+        }
+
+        // First verify the function is registered
+        const metadata = getFunctionMetadata("introspectTestFunc");
+        expect(metadata).toBeDefined();
+        expect(metadata?.name).toBe("introspecttestfunc");
+
+        // Use functions() with UNWIND to find the registered function
+        const runner = new Runner(`
+            WITH functions() AS funcs
+            UNWIND funcs AS f
+            WITH f WHERE f.name = 'introspecttestfunc'
+            RETURN f.name AS name, f.description AS description, f.category AS category
+        `);
+        await runner.run();
+        
+        expect(runner.results.length).toBe(1);
+        expect(runner.results[0].name).toBe("introspecttestfunc");
+        expect(runner.results[0].description).toBe("A unique test function for introspection");
+        expect(runner.results[0].category).toBe("scalar");
+    });
 });