flowquery 1.0.3 → 1.0.6

package/misc/apps/RAG/src/plugins/README.md

@@ -0,0 +1,139 @@
+# FlowQuery Plugin System
+
+This folder contains the plugin system for adding async data loader functions to FlowQuery.
+
+## Quick Start
+
+1. Create a new file in `loaders/` (e.g., `my-api.ts`)
+2. Define your plugin following the `AsyncLoaderPlugin` interface
+3. Import and add it to the `allPlugins` array in `index.ts`
+
+## Creating a Plugin
+
+### Basic Plugin Structure
+
+```typescript
+import { AsyncLoaderPlugin } from '../types';
+
+// Your async data provider function
+async function* myDataProvider(arg1: string, arg2?: number): AsyncGenerator<any, void, unknown> {
+    const response = await fetch(`https://api.example.com/${arg1}`);
+    const data = await response.json();
+    
+    // Yield items one at a time (for arrays)
+    if (Array.isArray(data)) {
+        for (const item of data) {
+            yield item;
+        }
+    } else {
+        yield data;
+    }
+}
+
+// Export the plugin definition
+export const myPlugin: AsyncLoaderPlugin = {
+    name: 'myData',  // Function name in FlowQuery
+    provider: myDataProvider,
+    metadata: {
+        description: 'Fetches data from My API',
+        category: 'data',
+        parameters: [
+            { name: 'arg1', description: 'First argument', type: 'string', required: true },
+            { name: 'arg2', description: 'Optional second arg', type: 'number', required: false }
+        ],
+        output: {
+            description: 'Data item',
+            type: 'object',
+            properties: {
+                id: { description: 'Item ID', type: 'number' },
+                value: { description: 'Item value', type: 'string' }
+            }
+        },
+        examples: [
+            "LOAD JSON FROM myData('users') AS item RETURN item.id, item.value"
+        ]
+    }
+};
+
+export default myPlugin;
+```
+
+### Register the Plugin
+
+Add your plugin to `index.ts`:
+
+```typescript
+import myPlugin from './loaders/my-api';
+
+const allPlugins: AsyncLoaderPlugin[] = [
+    // ... existing plugins
+    myPlugin,
+];
+```
+
+## Using Plugins in FlowQuery
+
+Once registered, use your plugin in FlowQuery queries:
+
+```sql
+-- Fetch data from your custom source
+LOAD JSON FROM myData('users') AS user
+WHERE user.active = true
+RETURN user.name, user.email
+
+-- Combine with filtering and ordering
+LOAD JSON FROM myData('products', 50) AS product
+WHERE product.price > 10
+ORDER BY product.name ASC
+RETURN product.name, product.price
+```
+
+## Available Built-in Plugins
+
+| Plugin | Description | Example |
+|--------|-------------|---------|
+| `fetchJson` | Fetch JSON from any URL | `fetchJson('https://api.example.com/data')` |
+
+> **Note**: Additional plugins may be registered. Check `index.ts` for the current list of available plugins.
+
+## Plugin Types
+
+### AsyncLoaderPlugin
+
+```typescript
+interface AsyncLoaderPlugin {
+    name: string;                    // Function name (lowercased when registered)
+    provider: AsyncDataProvider;     // The async generator/function
+    metadata?: PluginMetadata;       // Optional metadata for LLM consumption
+}
+```
+
+### AsyncDataProvider
+
+```typescript
+type AsyncDataProvider = (...args: any[]) => AsyncGenerator<any, void, unknown> | Promise<any>;
+```
+
+Plugins can return either:
+- An `AsyncGenerator` that yields items one at a time
+- A `Promise` that resolves to an array or single value
+
+### PluginMetadata
+
+```typescript
+interface PluginMetadata {
+    name: string;
+    description: string;
+    category?: string;
+    parameters?: ParameterSchema[];
+    output?: OutputSchema;
+    examples?: string[];
+}
+```
+
+## Tips
+
+1. **Use generators for large datasets** - Yield items one at a time to avoid memory issues
+2. **Add comprehensive metadata** - This helps LLMs understand and use your functions
+3. **Handle errors gracefully** - Throw descriptive errors for API failures
+4. **Include examples** - Show users how to use your plugin in FlowQuery

package/misc/apps/RAG/src/plugins/index.ts

@@ -0,0 +1,72 @@
+/**
+ * Plugin loader - automatically discovers and loads all plugins.
+ * 
+ * To add a new plugin:
+ * 1. Create a new file in the `loaders/` directory
+ * 2. Export your plugin(s) following the AsyncLoaderPlugin interface
+ * 3. Import and add to the plugins array in this file
+ */
+
+import pluginRegistry from './PluginRegistry';
+import { AsyncLoaderPlugin, PluginMetadata } from './types';
+
+// Import individual plugins
+import fetchJsonPlugin from './loaders/FetchJson';
+import catFactsPlugin from './loaders/CatFacts';
+import mockDataPlugins from './loaders/MockData';
+import llmPlugin from './loaders/Llm';
+
+/**
+ * All plugins to be loaded on startup.
+ * Add new plugins here as they are created.
+ */
+const allPlugins: AsyncLoaderPlugin[] = [
+    fetchJsonPlugin,
+    catFactsPlugin,
+    ...mockDataPlugins,
+    llmPlugin,
+];
+
+/**
+ * Initialize and load all plugins.
+ * Call this function once on app startup.
+ */
+export function initializePlugins(): void {
+    console.log('Initializing FlowQuery plugins...');
+    pluginRegistry.registerAll(allPlugins);
+    console.log(`Loaded ${pluginRegistry.getLoadedPlugins().length} plugins`);
+}
+
+/**
+ * Get the list of loaded plugin names.
+ * Uses FlowQuery's introspection to discover registered async providers.
+ */
+export function getLoadedPluginNames(): string[] {
+    return pluginRegistry.getLoadedPlugins();
+}
+
+/**
+ * Get metadata for all loaded plugins.
+ * Uses FlowQuery's functions() introspection as the single source of truth.
+ */
+export function getAllPluginMetadata(): PluginMetadata[] {
+    return pluginRegistry.getAllPluginMetadata();
+}
+
+/**
+ * Get all available async loader plugins by querying FlowQuery directly.
+ * This is the preferred async method that uses functions() introspection.
+ * 
+ * @returns Promise resolving to array of plugin metadata
+ */
+export async function getAvailableLoaders(): Promise<PluginMetadata[]> {
+    return pluginRegistry.getAvailableLoadersAsync();
+}
+
+// Re-export types and registry for external use
+export { pluginRegistry } from './PluginRegistry';
+export type { AsyncLoaderPlugin, PluginModule, PluginMetadata, AsyncDataProvider, ParameterSchema, OutputSchema, FunctionMetadata } from './types';
+
+// Re-export standalone loader functions for use outside of FlowQuery
+export { llm, llmStream, extractContent } from './loaders/Llm';
+export type { LlmOptions, LlmResponse } from './loaders/Llm';

package/misc/apps/RAG/src/plugins/loaders/CatFacts.ts

@@ -0,0 +1,79 @@
+/**
+ * Example plugin: Fetch random cat facts from the Cat Facts API.
+ * 
+ * Usage in FlowQuery:
+ *   LOAD JSON FROM catFacts(5) AS fact
+ *   RETURN fact.text
+ */
+
+import { AsyncLoaderPlugin } from '../types';
+
+const CAT_FACTS_API = 'https://catfact.ninja/facts';
+
+/**
+ * CatFacts loader class - fetches random cat facts from the Cat Facts API.
+ */
+export class CatFactsLoader {
+    private readonly apiUrl: string;
+
+    constructor(apiUrl: string = CAT_FACTS_API) {
+        this.apiUrl = apiUrl;
+    }
+
+    /**
+     * Fetches random cat facts from the Cat Facts API.
+     * 
+     * @param count - Number of cat facts to fetch (default: 1)
+     */
+    async *fetch(count: number = 1): AsyncGenerator<any, void, unknown> {
+        const url = `${this.apiUrl}?limit=${count}`;
+        const response = await fetch(url);
+        
+        if (!response.ok) {
+            throw new Error(`Failed to fetch cat facts: ${response.statusText}`);
+        }
+
+        const json = await response.json();
+        const data = json.data || [];
+        
+        for (const item of data) {
+            // Map 'fact' to 'text' for backwards compatibility with existing queries
+            yield {
+                text: item.fact,
+                length: item.length
+            };
+        }
+    }
+}
+
+export const catFactsPlugin: AsyncLoaderPlugin = {
+    name: 'catFacts',
+    provider: (count: number = 1) => new CatFactsLoader().fetch(count),
+    metadata: {
+        description: 'Fetches random cat facts from the Cat Facts API (catfact.ninja)',
+        category: 'examples',
+        parameters: [
+            {
+                name: 'count',
+                description: 'Number of cat facts to fetch',
+                type: 'number',
+                required: false,
+                default: 1
+            }
+        ],
+        output: {
+            description: 'Cat fact object',
+            type: 'object',
+            properties: {
+                text: { description: 'The cat fact text', type: 'string' },
+                length: { description: 'Length of the fact text', type: 'number' }
+            }
+        },
+        examples: [
+            "LOAD JSON FROM catFacts() AS fact RETURN fact.text",
+            "LOAD JSON FROM catFacts(5) AS fact RETURN fact.text, fact.length AS length"
+        ]
+    }
+};
+
+export default catFactsPlugin;

package/misc/apps/RAG/src/plugins/loaders/FetchJson.ts

@@ -0,0 +1,71 @@
+/**
+ * Example plugin: Fetch JSON data from a URL.
+ * 
+ * Usage in FlowQuery:
+ *   LOAD JSON FROM fetchJson('https://api.example.com/data') AS item
+ *   RETURN item.name, item.value
+ */
+
+import { AsyncLoaderPlugin } from '../types';
+
+/**
+ * FetchJson loader class - fetches JSON data from a URL and yields items.
+ */
+export class FetchJsonLoader {
+    /**
+     * Fetches JSON data from a URL and yields each item if array, or the object itself.
+     * 
+     * @param url - The URL to fetch JSON from
+     * @param options - Optional fetch options
+     */
+    async *fetch(url: string, options?: RequestInit): AsyncGenerator<any, void, unknown> {
+        const response = await fetch(url, options);
+        
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+
+        const data = await response.json();
+        
+        if (Array.isArray(data)) {
+            for (const item of data) {
+                yield item;
+            }
+        } else {
+            yield data;
+        }
+    }
+}
+
+export const fetchJsonPlugin: AsyncLoaderPlugin = {
+    name: 'fetchJson',
+    provider: (url: string, options?: RequestInit) => new FetchJsonLoader().fetch(url, options),
+    metadata: {
+        description: 'Fetches JSON data from a URL. If the response is an array, yields each item individually.',
+        category: 'data',
+        parameters: [
+            {
+                name: 'url',
+                description: 'The URL to fetch JSON from',
+                type: 'string',
+                required: true
+            },
+            {
+                name: 'options',
+                description: 'Optional fetch options (headers, method, etc.)',
+                type: 'object',
+                required: false
+            }
+        ],
+        output: {
+            description: 'JSON data items',
+            type: 'object'
+        },
+        examples: [
+            "LOAD JSON FROM fetchJson('https://api.example.com/users') AS user RETURN user.name",
+            "LOAD JSON FROM fetchJson('https://api.example.com/data') AS item RETURN item WHERE item.active = true"
+        ]
+    }
+};
+
+export default fetchJsonPlugin;