flowquery 1.0.5 → 1.0.7

package/misc/apps/RAG/src/plugins/loaders/MockData.ts

@@ -0,0 +1,151 @@
+/**
+ * Example plugin: Generate mock data for testing.
+ * 
+ * Usage in FlowQuery:
+ *   LOAD JSON FROM mockUsers(10) AS user
+ *   RETURN user.name, user.email
+ */
+
+import { FunctionDef } from 'flowquery/extensibility';
+
+/**
+ * MockUsers loader class - generates mock user data for testing.
+ */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Generates mock user data for testing purposes',
+    category: 'testing',
+    parameters: [
+        {
+            name: 'count',
+            description: 'Number of mock users to generate',
+            type: 'number',
+            required: false,
+            default: 5
+        }
+    ],
+    output: {
+        description: 'Mock user object',
+        type: 'object',
+        properties: {
+            id: { description: 'User ID', type: 'number' },
+            name: { description: 'Full name', type: 'string' },
+            email: { description: 'Email address', type: 'string' },
+            age: { description: 'Age in years', type: 'number' },
+            active: { description: 'Whether user is active', type: 'boolean' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM mockUsers(10) AS user RETURN user.name, user.email",
+        "LOAD JSON FROM mockUsers(20) AS user RETURN user WHERE user.active = true"
+    ]
+})
+export class MockUsersLoader {
+    private readonly firstNames: string[];
+    private readonly lastNames: string[];
+    private readonly domains: string[];
+
+    constructor(
+        firstNames: string[] = ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve', 'Frank', 'Grace', 'Henry', 'Ivy', 'Jack'],
+        lastNames: string[] = ['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia', 'Miller', 'Davis', 'Rodriguez', 'Martinez'],
+        domains: string[] = ['example.com', 'test.org', 'demo.net']
+    ) {
+        this.firstNames = firstNames;
+        this.lastNames = lastNames;
+        this.domains = domains;
+    }
+
+    /**
+     * Generates mock user data.
+     * 
+     * @param count - Number of mock users to generate
+     */
+    async *fetch(count: number = 5): AsyncGenerator<any, void, unknown> {
+        for (let i = 0; i < count; i++) {
+            const firstName = this.firstNames[Math.floor(Math.random() * this.firstNames.length)];
+            const lastName = this.lastNames[Math.floor(Math.random() * this.lastNames.length)];
+            const domain = this.domains[Math.floor(Math.random() * this.domains.length)];
+            
+            yield {
+                id: i + 1,
+                name: `${firstName} ${lastName}`,
+                email: `${firstName.toLowerCase()}.${lastName.toLowerCase()}@${domain}`,
+                age: Math.floor(Math.random() * 50) + 18,
+                active: Math.random() > 0.3
+            };
+        }
+    }
+}
+
+/**
+ * MockProducts loader class - generates mock product data for testing.
+ */
+@FunctionDef({
+    isAsyncProvider: true,
+    description: 'Generates mock product data for testing purposes',
+    category: 'testing',
+    parameters: [
+        {
+            name: 'count',
+            description: 'Number of mock products to generate',
+            type: 'number',
+            required: false,
+            default: 5
+        }
+    ],
+    output: {
+        description: 'Mock product object',
+        type: 'object',
+        properties: {
+            id: { description: 'Product ID', type: 'number' },
+            name: { description: 'Product name', type: 'string' },
+            category: { description: 'Product category', type: 'string' },
+            price: { description: 'Price in dollars', type: 'number' },
+            inStock: { description: 'Whether product is in stock', type: 'boolean' },
+            rating: { description: 'Customer rating (0-5)', type: 'number' }
+        }
+    },
+    examples: [
+        "LOAD JSON FROM mockProducts(10) AS p RETURN p.name, p.price",
+        "LOAD JSON FROM mockProducts(50) AS p RETURN p WHERE p.category = 'Electronics'"
+    ]
+})
+export class MockProductsLoader {
+    private readonly categories: string[];
+    private readonly adjectives: string[];
+    private readonly nouns: string[];
+
+    constructor(
+        categories: string[] = ['Electronics', 'Clothing', 'Books', 'Home', 'Sports'],
+        adjectives: string[] = ['Premium', 'Basic', 'Pro', 'Ultra', 'Classic'],
+        nouns: string[] = ['Widget', 'Gadget', 'Item', 'Product', 'Thing']
+    ) {
+        this.categories = categories;
+        this.adjectives = adjectives;
+        this.nouns = nouns;
+    }
+
+    /**
+     * Generates mock product data.
+     * 
+     * @param count - Number of mock products to generate
+     */
+    async *fetch(count: number = 5): AsyncGenerator<any, void, unknown> {
+        for (let i = 0; i < count; i++) {
+            const adj = this.adjectives[Math.floor(Math.random() * this.adjectives.length)];
+            const noun = this.nouns[Math.floor(Math.random() * this.nouns.length)];
+            const category = this.categories[Math.floor(Math.random() * this.categories.length)];
+            
+            yield {
+                id: i + 1,
+                name: `${adj} ${noun} ${i + 1}`,
+                category,
+                price: Math.round(Math.random() * 1000 * 100) / 100,
+                inStock: Math.random() > 0.2,
+                rating: Math.round(Math.random() * 50) / 10
+            };
+        }
+    }
+}
+
+export { MockUsersLoader as default };

package/misc/apps/RAG/src/prompts/FlowQuerySystemPrompt.ts

@@ -0,0 +1,385 @@
+/**
+ * FlowQuery System Prompt Generator
+ * 
+ * Generates a system prompt that instructs the LLM to create FlowQuery statements
+ * based on natural language queries, with awareness of available loader plugins.
+ * 
+ * Uses FlowQuery's built-in functions() introspection to dynamically discover
+ * available async data loaders and their metadata.
+ */
+
+import { FunctionMetadata, ParameterSchema, OutputSchema } from 'flowquery/extensibility';
+import { getAllPluginMetadata, getAvailableLoaders } from '../plugins';
+
+/**
+ * FlowQuery language reference documentation.
+ */
+const FLOWQUERY_LANGUAGE_REFERENCE = `
+## FlowQuery Language Reference
+
+FlowQuery is a declarative query language for data processing pipelines. It uses SQL-like syntax with additional constructs for working with APIs and data transformations.
+
+### Core Clauses
+
+1. **WITH** - Define variables and expressions
+   \`\`\`
+   WITH 'value' AS myVariable
+   WITH 1 + 2 AS result
+   WITH expression1 AS var1, expression2 AS var2
+   \`\`\`
+
+2. **LOAD JSON FROM** - Load data from a URL or async data provider
+   \`\`\`
+   LOAD JSON FROM 'https://api.example.com/data' AS item
+   LOAD JSON FROM myFunction(arg1, arg2) AS item
+   \`\`\`
+
+3. **LOAD JSON FROM ... HEADERS ... POST** - Make HTTP requests with headers and body
+   \`\`\`
+   LOAD JSON FROM 'https://api.example.com/data'
+   HEADERS {
+       \`Content-Type\`: 'application/json',
+       Authorization: f'Bearer {apiKey}'
+   }
+   POST {
+       field1: 'value1',
+       field2: variable
+   } AS response
+   \`\`\`
+
+4. **UNWIND** - Expand arrays into individual rows
+   \`\`\`
+   UNWIND [1, 2, 3] AS number
+   UNWIND myArray AS item
+   UNWIND range(0, 10) AS index
+   \`\`\`
+
+5. **WHERE** - Filter results
+   \`\`\`
+   WHERE item.active = true
+   WHERE user.age > 18 AND user.name CONTAINS 'John'
+   \`\`\`
+
+6. **RETURN** - Specify output columns
+   \`\`\`
+   RETURN item.name, item.value
+   RETURN item.name AS Name, item.price AS Price
+   RETURN *  -- Return all fields
+   \`\`\`
+
+7. **ORDER BY** - Sort results
+   \`\`\`
+   ORDER BY item.name ASC
+   ORDER BY item.price DESC, item.name ASC
+   \`\`\`
+
+8. **LIMIT** - Limit number of results
+   \`\`\`
+   LIMIT 10
+   \`\`\`
+
+9. **SKIP** - Skip a number of results
+   \`\`\`
+   SKIP 5
+   \`\`\`
+
+### Built-in Functions
+
+- **String Functions**: \`size()\`, \`substring()\`, \`trim()\`, \`toLower()\`, \`toUpper()\`, \`split()\`, \`join()\`, \`replace()\`, \`startsWith()\`, \`endsWith()\`, \`contains()\`
+- **Math Functions**: \`abs()\`, \`ceil()\`, \`floor()\`, \`round()\`, \`sqrt()\`, \`pow()\`, \`min()\`, \`max()\`
+- **Aggregate Functions**: \`sum()\`, \`avg()\`, \`count()\`, \`collect()\`, \`min()\`, \`max()\`
+- **List Functions**: \`range()\`, \`head()\`, \`tail()\`, \`last()\`, \`size()\`, \`reverse()\`
+- **Type Functions**: \`type()\`, \`toInteger()\`, \`toFloat()\`, \`toString()\`, \`toBoolean()\`
+- **Utility Functions**: \`coalesce()\`, \`keys()\`, \`properties()\`, \`stringify()\`
+
+### F-Strings (Template Literals)
+
+Use \`f"..."\` for string interpolation:
+\`\`\`
+WITH f"Hello, {name}!" AS greeting
+WITH f"The result is {value * 2}" AS message
+\`\`\`
+
+### Object and Array Literals
+
+\`\`\`
+WITH { name: 'John', age: 30 } AS person
+WITH [1, 2, 3] AS numbers
+WITH { items: [{ id: 1 }, { id: 2 }] } AS data
+\`\`\`
+
+### Property Access
+
+\`\`\`
+item.propertyName
+item['property-with-dashes']
+item.nested.property
+array[0]
+\`\`\`
+
+### Comparison Operators
+
+- \`=\`, \`<>\` (not equal), \`<\`, \`>\`, \`<=\`, \`>=\`
+- \`AND\`, \`OR\`, \`NOT\`
+- \`IN\`, \`CONTAINS\`, \`STARTS WITH\`, \`ENDS WITH\`
+- \`IS NULL\`, \`IS NOT NULL\`
+
+### Comments
+
+\`\`\`
+// Single line comment
+/* Multi-line
+   comment */
+\`\`\`
+`;
+
+/**
+ * FlowQuery System Prompt Generator class.
+ * Provides methods to generate various system prompts for LLM interactions.
+ */
+export class FlowQuerySystemPrompt {
+    /**
+     * Format a parameter schema into a readable string.
+     */
+    private static formatParameter(param: ParameterSchema): string {
+        const required = param.required ? ' (required)' : ' (optional)';
+        const defaultVal = param.default !== undefined ? `, default: ${JSON.stringify(param.default)}` : '';
+        const enumVals = param.enum ? `, values: [${param.enum.map(v => JSON.stringify(v)).join(', ')}]` : '';
+        return `  - \`${param.name}\`: ${param.type}${required}${defaultVal}${enumVals} - ${param.description}`;
+    }
+
+    /**
+     * Format output schema into a readable string.
+     */
+    private static formatOutput(output: OutputSchema): string {
+        let result = `  Returns: ${output.type} - ${output.description}`;
+        
+        if (output.properties) {
+            result += '\n  Output properties:';
+            for (const [key, prop] of Object.entries(output.properties)) {
+                result += `\n    - \`${key}\`: ${prop.type} - ${prop.description}`;
+            }
+        }
+        
+        if (output.example) {
+            result += `\n  Example output: ${JSON.stringify(output.example, null, 2)}`;
+        }
+        
+        return result;
+    }
+
+    /**
+     * Format a plugin metadata into a readable documentation block.
+     */
+    private static formatPluginDocumentation(plugin: FunctionMetadata): string {
+        const lines: string[] = [];
+        
+        lines.push(`### \`${plugin.name}\``);
+        lines.push(`**Description**: ${plugin.description}`);
+        
+        if (plugin.category) {
+            lines.push(`**Category**: ${plugin.category}`);
+        }
+        
+        if (plugin.parameters.length > 0) {
+            lines.push('\n**Parameters**:');
+            for (const param of plugin.parameters) {
+                lines.push(this.formatParameter(param));
+            }
+        } else {
+            lines.push('\n**Parameters**: None');
+        }
+        
+        lines.push('\n**Output**:');
+        lines.push(this.formatOutput(plugin.output));
+        
+        if (plugin.examples && plugin.examples.length > 0) {
+            lines.push('\n**Usage Examples**:');
+            for (const example of plugin.examples) {
+                lines.push(`\`\`\`\n${example}\n\`\`\``);
+            }
+        }
+        
+        if (plugin.notes) {
+            lines.push(`\n**Notes**: ${plugin.notes}`);
+        }
+        
+        return lines.join('\n');
+    }
+
+    /**
+     * Generate documentation for all available plugins.
+     */
+    private static generatePluginDocumentation(plugins: FunctionMetadata[]): string {
+        if (plugins.length === 0) {
+            return 'No data loader plugins are currently available.';
+        }
+        
+        const sections: string[] = [];
+        
+        // Group plugins by category
+        const byCategory = new Map<string, FunctionMetadata[]>();
+        for (const plugin of plugins) {
+            const category = plugin.category || 'general';
+            if (!byCategory.has(category)) {
+                byCategory.set(category, []);
+            }
+            byCategory.get(category)!.push(plugin);
+        }
+        
+        sections.push('## Available Data Loader Plugins\n');
+        sections.push('The following async data loader functions are available for use with `LOAD JSON FROM`:\n');
+        
+        for (const [category, categoryPlugins] of byCategory) {
+            sections.push(`\n### Category: ${category.charAt(0).toUpperCase() + category.slice(1)}\n`);
+            for (const plugin of categoryPlugins) {
+                sections.push(this.formatPluginDocumentation(plugin));
+                sections.push('---');
+            }
+        }
+        
+        return sections.join('\n');
+    }
+
+    /**
+     * Internal helper to build the system prompt from plugin documentation.
+     */
+    private static buildSystemPrompt(pluginDocs: string, additionalContext?: string): string {
+        return `You are a FlowQuery assistant. Your primary role is to help users by creating and executing FlowQuery statements based on their natural language requests.
+
+## How You Work
+
+You operate in a multi-step process:
+1. **Analyze** the user's natural language request
+2. **Generate** a FlowQuery statement that fulfills the request using available plugins
+3. The system will **execute** your FlowQuery and provide you with the results
+4. You will then **interpret** the results and present them to the user in a helpful way
+
+## Response Format for Query Generation
+
+When the user makes a request that requires fetching or processing data:
+1. Generate a FlowQuery statement wrapped in a code block with \`\`\`flowquery language tag
+2. Keep any explanation brief - the main focus should be the query
+3. The query will be automatically executed and you'll receive the results
+
+When the user asks a question that doesn't require data fetching (e.g., asking about FlowQuery syntax or general questions):
+1. Start your response with [NO_QUERY_NEEDED]
+2. Then provide your direct answer
+
+## Important Guidelines
+
+- Only use the available data loader plugins documented below
+- Use proper FlowQuery syntax as documented in the language reference
+- For API calls, prefer using the registered loader plugins over raw HTTP calls when possible
+- Always alias loaded items with \`AS\` for clarity
+- Use meaningful aliases in RETURN statements for better readability
+- Generate the simplest query that fulfills the user's request
+- If you cannot determine what the user needs, ask clarifying questions (with [NO_QUERY_NEEDED])
+
+${FLOWQUERY_LANGUAGE_REFERENCE}
+
+${pluginDocs}
+
+${additionalContext ? `## Additional Context\n\n${additionalContext}` : ''}
+
+## Example Response Format
+
+**When a query is needed**:
+\`\`\`flowquery
+LOAD JSON FROM pluginName(args) AS item
+WHERE item.field = 'value'
+RETURN item.name AS Name, item.value AS Value
+\`\`\`
+
+**When no query is needed** (e.g., general questions about FlowQuery):
+[NO_QUERY_NEEDED]
+Your direct answer here...
+
+Now help the user with their request.`;
+    }
+
+    /**
+     * Generate the complete FlowQuery system prompt.
+     * Uses FlowQuery's introspection via functions() as the single source of truth.
+     * 
+     * @param additionalContext - Optional additional context to include in the prompt
+     * @returns The complete system prompt string
+     */
+    public static generate(additionalContext?: string): string {
+        // Uses FlowQuery's introspection to get available async providers
+        const plugins = getAllPluginMetadata();
+        const pluginDocs = this.generatePluginDocumentation(plugins);
+        
+        return this.buildSystemPrompt(pluginDocs, additionalContext);
+    }
+
+    /**
+     * Generate a system prompt for the interpretation phase.
+     * Used after FlowQuery execution to interpret results.
+     * 
+     * @returns The interpretation system prompt string
+     */
+    public static generateInterpretationPrompt(): string {
+        return `You are a helpful assistant interpreting FlowQuery execution results.
+
+## Your Role
+
+The user made a natural language request, which was converted to a FlowQuery statement and executed.
+You are now receiving the execution results. Your job is to:
+
+1. **Summarize** the results in a clear, user-friendly way
+2. **Highlight** key insights or patterns in the data
+3. **Format** the data appropriately (tables, lists, or prose depending on the data)
+4. **Answer** the user's original question using the data
+
+## Guidelines
+
+- Be concise but thorough
+- If the results contain many items, summarize rather than listing all
+- If there's an error, explain what went wrong in user-friendly terms
+- Use markdown formatting for better readability
+- If the data doesn't fully answer the user's question, note what's missing`;
+    }
+
+    /**
+     * Get a minimal system prompt without full documentation.
+     * Useful for contexts where token count is a concern.
+     */
+    public static getMinimalPrompt(): string {
+        const plugins = getAllPluginMetadata();
+        const pluginList = plugins.map(p => `- \`${p.name}\`: ${p.description}`).join('\n');
+        
+        return `You are a FlowQuery assistant. Generate FlowQuery statements based on user requests.
+
+Available data loader plugins:
+${pluginList}
+
+FlowQuery uses SQL-like syntax: WITH, LOAD JSON FROM, UNWIND, WHERE, RETURN, ORDER BY, LIMIT, SKIP.
+Use f"..." for string interpolation. Access properties with dot notation or brackets.
+
+Always wrap FlowQuery code in \`\`\`flowquery code blocks.`;
+    }
+
+    /**
+     * Generate the FlowQuery system prompt asynchronously using functions() introspection.
+     * This is the preferred method that uses FlowQuery's built-in introspection.
+     * 
+     * @param additionalContext - Optional additional context to include in the prompt
+     * @returns Promise resolving to the complete system prompt string
+     */
+    public static async generateAsync(additionalContext?: string): Promise<string> {
+        // Use FlowQuery's functions() introspection to discover available loaders
+        const plugins = await getAvailableLoaders();
+        const pluginDocs = this.generatePluginDocumentation(plugins);
+        
+        return this.buildSystemPrompt(pluginDocs, additionalContext);
+    }
+}
+
+// Export functions for backward compatibility
+export const generateFlowQuerySystemPrompt = FlowQuerySystemPrompt.generate.bind(FlowQuerySystemPrompt);
+export const generateInterpretationPrompt = FlowQuerySystemPrompt.generateInterpretationPrompt.bind(FlowQuerySystemPrompt);
+export const getMinimalFlowQueryPrompt = FlowQuerySystemPrompt.getMinimalPrompt.bind(FlowQuerySystemPrompt);
+export const generateFlowQuerySystemPromptAsync = FlowQuerySystemPrompt.generateAsync.bind(FlowQuerySystemPrompt);
+
+export default FlowQuerySystemPrompt;

package/misc/apps/RAG/src/prompts/index.ts

@@ -0,0 +1,10 @@
+/**
+ * Prompts module - exports system prompt generators.
+ */
+
+export { 
+    FlowQuerySystemPrompt,
+    generateFlowQuerySystemPrompt, 
+    generateInterpretationPrompt,
+    getMinimalFlowQueryPrompt 
+} from './FlowQuerySystemPrompt';

package/misc/apps/RAG/src/utils/FlowQueryExecutor.ts

@@ -0,0 +1,131 @@
+/**
+ * FlowQuery Executor Utility
+ * 
+ * Executes FlowQuery statements and handles errors gracefully.
+ */
+
+import { FlowQuery } from 'flowquery';
+
+/**
+ * Result of executing a FlowQuery statement.
+ */
+export interface FlowQueryExecutionResult {
+    /** Whether the execution was successful */
+    success: boolean;
+    /** The query that was executed */
+    query: string;
+    /** The results from the query, if successful */
+    results?: any[];
+    /** Error message, if execution failed */
+    error?: string;
+    /** Execution time in milliseconds */
+    executionTime: number;
+}
+
+/**
+ * FlowQuery Executor class for executing FlowQuery statements.
+ * 
+ * @example
+ * ```typescript
+ * const executor = new FlowQueryExecutor();
+ * const result = await executor.execute("LOAD JSON FROM somePlugin(5) AS item RETURN item.text");
+ * if (result.success) {
+ *     console.log(result.results);
+ * } else {
+ *     console.error(result.error);
+ * }
+ * ```
+ */
+export class FlowQueryExecutor {
+    private defaultMaxItems: number;
+
+    /**
+     * Creates a new FlowQueryExecutor instance.
+     * @param defaultMaxItems - Default maximum items to display when formatting results (default: 20)
+     */
+    constructor(defaultMaxItems: number = 20) {
+        this.defaultMaxItems = defaultMaxItems;
+    }
+
+    /**
+     * Execute a FlowQuery statement and return the results.
+     * 
+     * @param query - The FlowQuery statement to execute
+     * @returns The execution result including success status, results or error
+     */
+    async execute(query: string): Promise<FlowQueryExecutionResult> {
+        const startTime = performance.now();
+        
+        try {
+            // Validate the query is not empty
+            if (!query || query.trim() === '') {
+                return {
+                    success: false,
+                    query,
+                    error: 'Query cannot be empty',
+                    executionTime: performance.now() - startTime
+                };
+            }
+
+            // Create a runner and execute the query
+            const runner = new FlowQuery(query);
+            await runner.run();
+            
+            // Get the results
+            const results = runner.results;
+            
+            return {
+                success: true,
+                query,
+                results: Array.isArray(results) ? results : [results],
+                executionTime: performance.now() - startTime
+            };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            
+            return {
+                success: false,
+                query,
+                error: errorMessage,
+                executionTime: performance.now() - startTime
+            };
+        }
+    }
+
+    /**
+     * Format execution results for display or LLM consumption.
+     * 
+     * @param result - The execution result to format
+     * @param maxItems - Maximum number of items to include (uses default if not specified)
+     * @returns A formatted string representation of the results
+     */
+    formatResult(result: FlowQueryExecutionResult, maxItems?: number): string {
+        const limit = maxItems ?? this.defaultMaxItems;
+
+        if (!result.success) {
+            return `Execution Error: ${result.error}`;
+        }
+
+        const results = result.results || [];
+        const totalCount = results.length;
+        const displayResults = results.slice(0, limit);
+        
+        let output = `Execution successful (${result.executionTime.toFixed(2)}ms)\n`;
+        output += `Total results: ${totalCount}\n\n`;
+        
+        if (totalCount === 0) {
+            output += 'No results returned.';
+        } else {
+            output += 'Results:\n';
+            output += JSON.stringify(displayResults, null, 2);
+            
+            if (totalCount > limit) {
+                output += `\n\n... and ${totalCount - limit} more results`;
+            }
+        }
+        
+        return output;
+    }
+}
+
+export default FlowQueryExecutor;