flowquery 1.0.6 → 1.0.8

package/misc/apps/RAG/src/plugins/loaders/CatFacts.ts

@@ -6,13 +6,39 @@
  *   RETURN fact.text
  */
 
-import { AsyncLoaderPlugin } from '../types';
+import { FunctionDef } from 'flowquery/extensibility';
 
 const CAT_FACTS_API = 'https://catfact.ninja/facts';
 
 /**
  * CatFacts loader class - fetches random cat facts from the Cat Facts API.
  */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Fetches random cat facts from the Cat Facts API (catfact.ninja)',
+    category: 'examples',
+    parameters: [
+        {
+            name: 'count',
+            description: 'Number of cat facts to fetch',
+            type: 'number',
+            required: false,
+            default: 1
+        }
+    ],
+    output: {
+        description: 'Cat fact object',
+        type: 'object',
+        properties: {
+            text: { description: 'The cat fact text', type: 'string' },
+            length: { description: 'Length of the fact text', type: 'number' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM catFacts() AS fact RETURN fact.text",
+        "LOAD JSON FROM catFacts(5) AS fact RETURN fact.text, fact.length AS length"
+    ]
+})
 export class CatFactsLoader {
     private readonly apiUrl: string;
 
@@ -46,34 +72,4 @@ export class CatFactsLoader {
     }
 }
 
-export const catFactsPlugin: AsyncLoaderPlugin = {
-    name: 'catFacts',
-    provider: (count: number = 1) => new CatFactsLoader().fetch(count),
-    metadata: {
-        description: 'Fetches random cat facts from the Cat Facts API (catfact.ninja)',
-        category: 'examples',
-        parameters: [
-            {
-                name: 'count',
-                description: 'Number of cat facts to fetch',
-                type: 'number',
-                required: false,
-                default: 1
-            }
-        ],
-        output: {
-            description: 'Cat fact object',
-            type: 'object',
-            properties: {
-                text: { description: 'The cat fact text', type: 'string' },
-                length: { description: 'Length of the fact text', type: 'number' }
-            }
-        },
-        examples: [
-            "LOAD JSON FROM catFacts() AS fact RETURN fact.text",
-            "LOAD JSON FROM catFacts(5) AS fact RETURN fact.text, fact.length AS length"
-        ]
-    }
-};
-
-export default catFactsPlugin;
+export default CatFactsLoader;

package/misc/apps/RAG/src/plugins/loaders/FetchJson.ts

@@ -6,11 +6,38 @@
  *   RETURN item.name, item.value
  */
 
-import { AsyncLoaderPlugin } from '../types';
+import { FunctionDef } from 'flowquery/extensibility';
 
 /**
  * FetchJson loader class - fetches JSON data from a URL and yields items.
  */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Fetches JSON data from a URL. If the response is an array, yields each item individually.',
+    category: 'data',
+    parameters: [
+        {
+            name: 'url',
+            description: 'The URL to fetch JSON from',
+            type: 'string',
+            required: true
+        },
+        {
+            name: 'options',
+            description: 'Optional fetch options (headers, method, etc.)',
+            type: 'object',
+            required: false
+        }
+    ],
+    output: {
+        description: 'JSON data items',
+        type: 'object'
+    },
+    examples: [
+        "LOAD JSON FROM fetchJson('https://api.example.com/users') AS user RETURN user.name",
+        "LOAD JSON FROM fetchJson('https://api.example.com/data') AS item RETURN item WHERE item.active = true"
+    ]
+})
 export class FetchJsonLoader {
     /**
      * Fetches JSON data from a URL and yields each item if array, or the object itself.
@@ -37,35 +64,4 @@ export class FetchJsonLoader {
     }
 }
 
-export const fetchJsonPlugin: AsyncLoaderPlugin = {
-    name: 'fetchJson',
-    provider: (url: string, options?: RequestInit) => new FetchJsonLoader().fetch(url, options),
-    metadata: {
-        description: 'Fetches JSON data from a URL. If the response is an array, yields each item individually.',
-        category: 'data',
-        parameters: [
-            {
-                name: 'url',
-                description: 'The URL to fetch JSON from',
-                type: 'string',
-                required: true
-            },
-            {
-                name: 'options',
-                description: 'Optional fetch options (headers, method, etc.)',
-                type: 'object',
-                required: false
-            }
-        ],
-        output: {
-            description: 'JSON data items',
-            type: 'object'
-        },
-        examples: [
-            "LOAD JSON FROM fetchJson('https://api.example.com/users') AS user RETURN user.name",
-            "LOAD JSON FROM fetchJson('https://api.example.com/data') AS item RETURN item WHERE item.active = true"
-        ]
-    }
-};
-
-export default fetchJsonPlugin;
+export default FetchJsonLoader;

package/misc/apps/RAG/src/plugins/loaders/Llm.ts

@@ -16,7 +16,7 @@
  *   console.log(response.choices[0].message.content);
  */
 
-import { AsyncLoaderPlugin } from '../types';
+import { FunctionDef } from 'flowquery/extensibility';
 
 // Default configuration - can be overridden via options
 const DEFAULT_CONFIG = {
@@ -80,6 +80,59 @@ export interface LlmResponse {
 /**
  * LLM Loader class - calls OpenAI-compatible APIs for chat completions.
  */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Calls OpenAI-compatible chat completion APIs. Supports GPT models and any OpenAI-compatible endpoint.',
+    category: 'ai',
+    parameters: [
+        {
+            name: 'prompt',
+            description: 'The user prompt to send to the LLM',
+            type: 'string',
+            required: true,
+            example: 'What is the capital of France?'
+        },
+        {
+            name: 'options',
+            description: 'Optional configuration for the LLM request',
+            type: 'object',
+            required: false,
+            properties: {
+                apiKey: { description: 'OpenAI API key', type: 'string' },
+                apiUrl: { description: 'API endpoint URL (defaults to OpenAI chat completions)', type: 'string' },
+                model: { description: 'Model to use (defaults to gpt-4o-mini)', type: 'string' },
+                temperature: { description: 'Sampling temperature 0-2 (defaults to 0.7)', type: 'number' },
+                maxTokens: { description: 'Maximum tokens to generate', type: 'number' },
+                systemPrompt: { description: 'System prompt to set context', type: 'string' },
+                messages: { description: 'Additional conversation messages', type: 'array' },
+                organizationId: { description: 'OpenAI organization ID', type: 'string' },
+                headers: { description: 'Additional request headers', type: 'object' },
+                stream: { description: 'Enable streaming response', type: 'boolean' },
+                additionalParams: { description: 'Additional API parameters', type: 'object' }
+            }
+        }
+    ],
+    output: {
+        description: 'OpenAI chat completion response',
+        type: 'object',
+        properties: {
+            id: { description: 'Unique identifier for the completion', type: 'string' },
+            model: { description: 'Model used for completion', type: 'string' },
+            choices: { 
+                description: 'Array of completion choices', 
+                type: 'array',
+                example: [{ message: { role: 'assistant', content: 'Paris is the capital of France.' } }]
+            },
+            usage: { description: 'Token usage statistics', type: 'object' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM llm('What is 2+2?') AS response RETURN response.choices[0].message.content",
+        "LOAD JSON FROM llm('Translate to French: Hello', { model: 'gpt-4o', temperature: 0.3 }) AS response RETURN response.choices[0].message.content",
+        "LOAD JSON FROM llm('Write a haiku', { systemPrompt: 'You are a poet' }) AS response RETURN response.choices[0].message.content"
+    ],
+    notes: 'Requires API key configured in Settings or passed as apiKey option. Works with any OpenAI-compatible API by setting the apiUrl option.'
+})
 export class LlmLoader {
     private readonly defaultOptions: Partial<LlmOptions>;
 
@@ -381,61 +434,4 @@ export function extractContent(response: LlmResponse): string {
     return LlmLoader.extractContent(response);
 }
 
-export const llmPlugin: AsyncLoaderPlugin = {
-    name: 'llm',
-    provider: (prompt: string, options?: LlmOptions) => new LlmLoader().fetch(prompt, options),
-    metadata: {
-        description: 'Calls OpenAI-compatible chat completion APIs. Supports GPT models and any OpenAI-compatible endpoint.',
-        category: 'ai',
-        parameters: [
-            {
-                name: 'prompt',
-                description: 'The user prompt to send to the LLM',
-                type: 'string',
-                required: true,
-                example: 'What is the capital of France?'
-            },
-            {
-                name: 'options',
-                description: 'Optional configuration for the LLM request',
-                type: 'object',
-                required: false,
-                properties: {
-                    apiKey: { description: 'OpenAI API key', type: 'string' },
-                    apiUrl: { description: 'API endpoint URL (defaults to OpenAI chat completions)', type: 'string' },
-                    model: { description: 'Model to use (defaults to gpt-4o-mini)', type: 'string' },
-                    temperature: { description: 'Sampling temperature 0-2 (defaults to 0.7)', type: 'number' },
-                    maxTokens: { description: 'Maximum tokens to generate', type: 'number' },
-                    systemPrompt: { description: 'System prompt to set context', type: 'string' },
-                    messages: { description: 'Additional conversation messages', type: 'array' },
-                    organizationId: { description: 'OpenAI organization ID', type: 'string' },
-                    headers: { description: 'Additional request headers', type: 'object' },
-                    stream: { description: 'Enable streaming response', type: 'boolean' },
-                    additionalParams: { description: 'Additional API parameters', type: 'object' }
-                }
-            }
-        ],
-        output: {
-            description: 'OpenAI chat completion response',
-            type: 'object',
-            properties: {
-                id: { description: 'Unique identifier for the completion', type: 'string' },
-                model: { description: 'Model used for completion', type: 'string' },
-                choices: { 
-                    description: 'Array of completion choices', 
-                    type: 'array',
-                    example: [{ message: { role: 'assistant', content: 'Paris is the capital of France.' } }]
-                },
-                usage: { description: 'Token usage statistics', type: 'object' }
-            }
-        },
-        examples: [
-            "LOAD JSON FROM llm('What is 2+2?') AS response RETURN response.choices[0].message.content",
-            "LOAD JSON FROM llm('Translate to French: Hello', { model: 'gpt-4o', temperature: 0.3 }) AS response RETURN response.choices[0].message.content",
-            "LOAD JSON FROM llm('Write a haiku', { systemPrompt: 'You are a poet' }) AS response RETURN response.choices[0].message.content"
-        ],
-        notes: 'Requires API key configured in Settings or passed as apiKey option. Works with any OpenAI-compatible API by setting the apiUrl option.'
-    }
-};
-
-export default llmPlugin;
+export default LlmLoader;

package/misc/apps/RAG/src/plugins/loaders/MockData.ts

@@ -6,11 +6,40 @@
  *   RETURN user.name, user.email
  */
 
-import { AsyncLoaderPlugin } from '../types';
+import { FunctionDef } from 'flowquery/extensibility';
 
 /**
  * MockUsers loader class - generates mock user data for testing.
  */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Generates mock user data for testing purposes',
+    category: 'testing',
+    parameters: [
+        {
+            name: 'count',
+            description: 'Number of mock users to generate',
+            type: 'number',
+            required: false,
+            default: 5
+        }
+    ],
+    output: {
+        description: 'Mock user object',
+        type: 'object',
+        properties: {
+            id: { description: 'User ID', type: 'number' },
+            name: { description: 'Full name', type: 'string' },
+            email: { description: 'Email address', type: 'string' },
+            age: { description: 'Age in years', type: 'number' },
+            active: { description: 'Whether user is active', type: 'boolean' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM mockUsers(10) AS user RETURN user.name, user.email",
+        "LOAD JSON FROM mockUsers(20) AS user RETURN user WHERE user.active = true"
+    ]
+})
 export class MockUsersLoader {
     private readonly firstNames: string[];
     private readonly lastNames: string[];
@@ -51,6 +80,36 @@ export class MockUsersLoader {
 /**
  * MockProducts loader class - generates mock product data for testing.
  */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Generates mock product data for testing purposes',
+    category: 'testing',
+    parameters: [
+        {
+            name: 'count',
+            description: 'Number of mock products to generate',
+            type: 'number',
+            required: false,
+            default: 5
+        }
+    ],
+    output: {
+        description: 'Mock product object',
+        type: 'object',
+        properties: {
+            id: { description: 'Product ID', type: 'number' },
+            name: { description: 'Product name', type: 'string' },
+            category: { description: 'Product category', type: 'string' },
+            price: { description: 'Price in dollars', type: 'number' },
+            inStock: { description: 'Whether product is in stock', type: 'boolean' },
+            rating: { description: 'Customer rating (0-5)', type: 'number' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM mockProducts(10) AS p RETURN p.name, p.price",
+        "LOAD JSON FROM mockProducts(50) AS p RETURN p WHERE p.category = 'Electronics'"
+    ]
+})
 export class MockProductsLoader {
     private readonly categories: string[];
     private readonly adjectives: string[];
@@ -89,73 +148,4 @@ export class MockProductsLoader {
     }
 }
 
-export const mockUsersPlugin: AsyncLoaderPlugin = {
-    name: 'mockUsers',
-    provider: (count: number = 5) => new MockUsersLoader().fetch(count),
-    metadata: {
-        description: 'Generates mock user data for testing purposes',
-        category: 'testing',
-        parameters: [
-            {
-                name: 'count',
-                description: 'Number of mock users to generate',
-                type: 'number',
-                required: false,
-                default: 5
-            }
-        ],
-        output: {
-            description: 'Mock user object',
-            type: 'object',
-            properties: {
-                id: { description: 'User ID', type: 'number' },
-                name: { description: 'Full name', type: 'string' },
-                email: { description: 'Email address', type: 'string' },
-                age: { description: 'Age in years', type: 'number' },
-                active: { description: 'Whether user is active', type: 'boolean' }
-            }
-        },
-        examples: [
-            "LOAD JSON FROM mockUsers(10) AS user RETURN user.name, user.email",
-            "LOAD JSON FROM mockUsers(20) AS user RETURN user WHERE user.active = true"
-        ]
-    }
-};
-
-export const mockProductsPlugin: AsyncLoaderPlugin = {
-    name: 'mockProducts',
-    provider: (count: number = 5) => new MockProductsLoader().fetch(count),
-    metadata: {
-        description: 'Generates mock product data for testing purposes',
-        category: 'testing',
-        parameters: [
-            {
-                name: 'count',
-                description: 'Number of mock products to generate',
-                type: 'number',
-                required: false,
-                default: 5
-            }
-        ],
-        output: {
-            description: 'Mock product object',
-            type: 'object',
-            properties: {
-                id: { description: 'Product ID', type: 'number' },
-                name: { description: 'Product name', type: 'string' },
-                category: { description: 'Product category', type: 'string' },
-                price: { description: 'Price in dollars', type: 'number' },
-                inStock: { description: 'Whether product is in stock', type: 'boolean' },
-                rating: { description: 'Customer rating (0-5)', type: 'number' }
-            }
-        },
-        examples: [
-            "LOAD JSON FROM mockProducts(10) AS p RETURN p.name, p.price",
-            "LOAD JSON FROM mockProducts(50) AS p RETURN p WHERE p.category = 'Electronics'"
-        ]
-    }
-};
-
-export const plugins = [mockUsersPlugin, mockProductsPlugin];
-
-export default plugins;
+export { MockUsersLoader as default };

package/misc/apps/RAG/src/prompts/FlowQuerySystemPrompt.ts

@@ -8,8 +8,8 @@
  * available async data loaders and their metadata.
  */
 
-import { PluginMetadata, ParameterSchema, OutputSchema } from '../plugins/types';
-import { pluginRegistry } from '../plugins/PluginRegistry';
+import { FunctionMetadata, ParameterSchema, OutputSchema } from 'flowquery/extensibility';
+import { getAllPluginMetadata, getAvailableLoaders } from '../plugins';
 
 /**
  * FlowQuery language reference documentation.
@@ -171,7 +171,7 @@ export class FlowQuerySystemPrompt {
     /**
      * Format a plugin metadata into a readable documentation block.
      */
-    private static formatPluginDocumentation(plugin: PluginMetadata): string {
+    private static formatPluginDocumentation(plugin: FunctionMetadata): string {
         const lines: string[] = [];
         
         lines.push(`### \`${plugin.name}\``);
@@ -210,7 +210,7 @@ export class FlowQuerySystemPrompt {
     /**
      * Generate documentation for all available plugins.
      */
-    private static generatePluginDocumentation(plugins: PluginMetadata[]): string {
+    private static generatePluginDocumentation(plugins: FunctionMetadata[]): string {
         if (plugins.length === 0) {
             return 'No data loader plugins are currently available.';
         }
@@ -218,7 +218,7 @@ export class FlowQuerySystemPrompt {
         const sections: string[] = [];
         
         // Group plugins by category
-        const byCategory = new Map<string, PluginMetadata[]>();
+        const byCategory = new Map<string, FunctionMetadata[]>();
         for (const plugin of plugins) {
             const category = plugin.category || 'general';
             if (!byCategory.has(category)) {
@@ -307,7 +307,7 @@ Now help the user with their request.`;
      */
     public static generate(additionalContext?: string): string {
         // Uses FlowQuery's introspection to get available async providers
-        const plugins = pluginRegistry.getAllPluginMetadata();
+        const plugins = getAllPluginMetadata();
         const pluginDocs = this.generatePluginDocumentation(plugins);
         
         return this.buildSystemPrompt(pluginDocs, additionalContext);
@@ -346,7 +346,7 @@ You are now receiving the execution results. Your job is to:
      * Useful for contexts where token count is a concern.
      */
     public static getMinimalPrompt(): string {
-        const plugins = pluginRegistry.getAllPluginMetadata();
+        const plugins = getAllPluginMetadata();
         const pluginList = plugins.map(p => `- \`${p.name}\`: ${p.description}`).join('\n');
         
         return `You are a FlowQuery assistant. Generate FlowQuery statements based on user requests.
@@ -369,7 +369,7 @@ Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
      */
     public static async generateAsync(additionalContext?: string): Promise<string> {
         // Use FlowQuery's functions() introspection to discover available loaders
-        const plugins = await pluginRegistry.getAvailableLoadersAsync();
+        const plugins = await getAvailableLoaders();
         const pluginDocs = this.generatePluginDocumentation(plugins);
         
         return this.buildSystemPrompt(pluginDocs, additionalContext);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",

package/src/compute/runner.ts

@@ -1,7 +1,6 @@
 import Operation from "../parsing/operations/operation";
 import Parser from "../parsing/parser";
-import { FunctionCreator, AsyncDataProvider } from "../parsing/functions/function_factory";
-import { FunctionMetadata, RegisterFunctionOptions, RegisterAsyncProviderOptions } from "../parsing/functions/function_metadata";
+import { FunctionMetadata } from "../parsing/functions/function_metadata";
 import Function from "../parsing/functions/function";
 
 /**
@@ -21,30 +20,6 @@ class Runner {
     private first: Operation;
     private last: Operation;
 
-    /**
-     * Register a synchronous plugin function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static registerFunction: (name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions) => void;
-
-    /**
-     * Unregister a synchronous plugin function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static unregisterFunction: (name: string) => void;
-
-    /**
-     * Register an async data provider function for use in LOAD operations.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static registerAsyncProvider: (name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions) => void;
-
-    /**
-     * Unregister an async data provider function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static unregisterAsyncProvider: (name: string) => void;
-
     /**
      * List all registered functions with their metadata.
      * Added dynamically in index.browser.ts / index.node.ts

package/src/extensibility.ts

@@ -1,9 +1,45 @@
 /**
  * FlowQuery Extensibility API
  * 
- * This module re-exports the extensibility API from the parsing/functions/extensibility module.
+ * This module provides all the exports needed to create custom FlowQuery functions.
  * 
  * @packageDocumentation
+ * 
+ * @example
+ * ```typescript
+ * import { Function, FunctionDef } from 'flowquery/extensibility';
+ * 
+ * @FunctionDef({
+ *     description: "Converts a string to uppercase",
+ *     category: "string",
+ *     parameters: [{ name: "text", description: "String to convert", type: "string" }],
+ *     output: { description: "Uppercase string", type: "string" }
+ * })
+ * class UpperCase extends Function {
+ *     constructor() {
+ *         super("uppercase");
+ *         this._expectedParameterCount = 1;
+ *     }
+ *     public value(): string {
+ *         return String(this.getChildren()[0].value()).toUpperCase();
+ *     }
+ * }
+ * ```
  */
 
-export * from "./parsing/functions/extensibility";
+// Base function classes for creating custom functions
+export { default as Function } from "./parsing/functions/function";
+export { default as AggregateFunction } from "./parsing/functions/aggregate_function";
+export { default as AsyncFunction } from "./parsing/functions/async_function";
+export { default as PredicateFunction } from "./parsing/functions/predicate_function";
+export { default as ReducerElement } from "./parsing/functions/reducer_element";
+
+// Decorator and metadata types for function registration
+export { 
+    FunctionDef,
+    FunctionMetadata,
+    FunctionDefOptions,
+    ParameterSchema,
+    OutputSchema,
+    FunctionCategory
+} from "./parsing/functions/function_metadata";