flowquery 1.0.5 → 1.0.6

package/misc/apps/RAG/src/utils/FlowQueryExtractor.ts

@@ -0,0 +1,203 @@
+/**
+ * FlowQuery Extraction Utility
+ * 
+ * Extracts FlowQuery statements from LLM responses.
+ */
+
+/**
+ * Extraction result containing the query and any explanation text.
+ */
+export interface FlowQueryExtraction {
+    /** The extracted FlowQuery statement, if found */
+    query: string | null;
+    /** Whether a query was successfully extracted */
+    found: boolean;
+    /** Any explanation text before the code block */
+    explanation?: string;
+    /** Whether the LLM indicated no query is needed */
+    noQueryNeeded?: boolean;
+    /** Message from the LLM if no query is needed */
+    directResponse?: string;
+}
+
+/**
+ * FlowQuery Extractor class for extracting FlowQuery statements from LLM responses.
+ * 
+ * Looks for code blocks with flowquery, cypher, or sql language tags,
+ * or generic code blocks that appear to contain FlowQuery syntax.
+ * 
+ * @example
+ * ```typescript
+ * const extractor = new FlowQueryExtractor();
+ * const response = `Here's the query:
+ * \`\`\`flowquery
+ * LOAD JSON FROM somePlugin(5) AS item
+ * RETURN item.text
+ * \`\`\``;
+ * 
+ * const extraction = extractor.extract(response);
+ * console.log(extraction.query); // "LOAD JSON FROM somePlugin(5) AS item\nRETURN item.text"
+ * ```
+ */
+export class FlowQueryExtractor {
+    /** Regex patterns for matching code blocks with language tags */
+    private readonly codeBlockPatterns: RegExp[] = [
+        /```(?:flowquery|cypher|fql)\s*\n([\s\S]*?)```/i,
+        /```(?:sql)\s*\n([\s\S]*?)```/i,
+        /```\s*\n([\s\S]*?)```/,  // Generic code block
+    ];
+
+    /** Keywords that indicate a FlowQuery statement */
+    private readonly flowQueryKeywords: RegExp[] = [
+        /\bWITH\b/i,
+        /\bLOAD\s+JSON\s+FROM\b/i,
+        /\bUNWIND\b/i,
+        /\bRETURN\b/i,
+        /\bWHERE\b/i,
+        /\bORDER\s+BY\b/i,
+        /\bLIMIT\b/i,
+    ];
+
+    /** Keywords that can start a FlowQuery statement */
+    private readonly startKeywords = /^(WITH|LOAD\s+JSON\s+FROM|UNWIND)\b/i;
+
+    /** Keywords that can continue a FlowQuery statement */
+    private readonly continueKeywords = /^(WITH|LOAD|UNWIND|WHERE|RETURN|ORDER|LIMIT|SKIP|HEADERS|POST|AS)\b/i;
+
+    /**
+     * Extract a FlowQuery statement from an LLM response.
+     * 
+     * @param llmResponse - The full text response from the LLM
+     * @returns The extraction result
+     */
+    public extract(llmResponse: string): FlowQueryExtraction {
+        if (!llmResponse || llmResponse.trim() === '') {
+            return { query: null, found: false };
+        }
+
+        // Check for explicit "NO_QUERY_NEEDED" marker
+        if (llmResponse.includes('[NO_QUERY_NEEDED]') || 
+            llmResponse.includes('NO_QUERY_NEEDED')) {
+            // Extract the direct response after the marker
+            const directMatch = llmResponse.match(/\[NO_QUERY_NEEDED\]\s*([\s\S]*)/i);
+            return {
+                query: null,
+                found: false,
+                noQueryNeeded: true,
+                directResponse: directMatch ? directMatch[1].trim() : llmResponse
+            };
+        }
+
+        // Try to match code blocks with specific language tags
+        for (const pattern of this.codeBlockPatterns) {
+            const match = llmResponse.match(pattern);
+            if (match && match[1]) {
+                const query = match[1].trim();
+                
+                // Verify it looks like a FlowQuery statement
+                if (this.isLikelyFlowQuery(query)) {
+                    // Extract explanation text before the code block
+                    const beforeMatch = llmResponse.substring(0, llmResponse.indexOf(match[0])).trim();
+                    
+                    return {
+                        query,
+                        found: true,
+                        explanation: beforeMatch || undefined
+                    };
+                }
+            }
+        }
+
+        // Try to extract inline FlowQuery if no code block found
+        // Look for lines starting with FlowQuery keywords
+        const inlineQuery = this.extractInlineQuery(llmResponse);
+        if (inlineQuery) {
+            return {
+                query: inlineQuery,
+                found: true
+            };
+        }
+
+        return { query: null, found: false };
+    }
+
+    /**
+     * Extract multiple FlowQuery statements from an LLM response.
+     * Useful when the LLM provides alternative queries.
+     * 
+     * @param llmResponse - The full text response from the LLM
+     * @returns Array of extracted queries
+     */
+    public extractAll(llmResponse: string): string[] {
+        if (!llmResponse) return [];
+
+        const queries: string[] = [];
+        const codeBlockPattern = /```(?:flowquery|cypher|fql|sql)?\s*\n([\s\S]*?)```/gi;
+        
+        let match;
+        while ((match = codeBlockPattern.exec(llmResponse)) !== null) {
+            if (match[1]) {
+                const query = match[1].trim();
+                if (this.isLikelyFlowQuery(query)) {
+                    queries.push(query);
+                }
+            }
+        }
+
+        return queries;
+    }
+
+    /**
+     * Check if a string looks like a FlowQuery statement.
+     */
+    private isLikelyFlowQuery(text: string): boolean {
+        // Must contain at least one FlowQuery keyword
+        return this.flowQueryKeywords.some(pattern => pattern.test(text));
+    }
+
+    /**
+     * Try to extract a FlowQuery statement that's not in a code block.
+     */
+    private extractInlineQuery(text: string): string | null {
+        const lines = text.split('\n');
+        const queryLines: string[] = [];
+        let inQuery = false;
+
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            
+            if (!inQuery && this.startKeywords.test(trimmedLine)) {
+                inQuery = true;
+                queryLines.push(trimmedLine);
+            } else if (inQuery) {
+                // Check if this line continues the query
+                if (this.continueKeywords.test(trimmedLine) || 
+                    trimmedLine.startsWith('{') ||
+                    trimmedLine.startsWith('}') ||
+                    trimmedLine === '' ||
+                    /^[A-Za-z_][A-Za-z0-9_]*\./.test(trimmedLine) ||
+                    /^\s+/.test(line)) {  // Indented line
+                    queryLines.push(trimmedLine);
+                } else {
+                    // End of query
+                    break;
+                }
+            }
+        }
+
+        const query = queryLines.join('\n').trim();
+        return query && this.isLikelyFlowQuery(query) ? query : null;
+    }
+}
+
+// Convenience function for backward compatibility
+export function extractFlowQuery(llmResponse: string): FlowQueryExtraction {
+    return new FlowQueryExtractor().extract(llmResponse);
+}
+
+// Convenience function for backward compatibility
+export function extractAllFlowQueries(llmResponse: string): string[] {
+    return new FlowQueryExtractor().extractAll(llmResponse);
+}
+
+export default FlowQueryExtractor;

package/misc/apps/RAG/src/utils/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Utils module exports
+ */
+
+export { FlowQueryExecutor } from './FlowQueryExecutor';
+export type { FlowQueryExecutionResult } from './FlowQueryExecutor';
+
+export { FlowQueryExtractor, extractFlowQuery, extractAllFlowQueries } from './FlowQueryExtractor';
+export type { FlowQueryExtraction } from './FlowQueryExtractor';

package/misc/apps/RAG/tsconfig.json

@@ -3,8 +3,8 @@
     "outDir": "./dist",
     "rootDir": "./src",
     "target": "ES2020",
-    "module": "commonjs",
-    "moduleResolution": "node",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "strict": true,
@@ -12,6 +12,8 @@
     "declaration": true,
     "sourceMap": true,
     "resolveJsonModule": true,
+    "jsx": "react-jsx",
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
     "types": ["node"]
   },
   "include": ["src/**/*"],

package/misc/apps/RAG/webpack.config.js

@@ -1,32 +1,43 @@
 const path = require('path');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
 
 module.exports = {
-  mode: 'production',
-  entry: './src/index.ts',
-  target: 'node',
+  mode: 'development',
+  entry: './src/index.tsx',
+  target: 'web',
   output: {
-    filename: 'rag.bundle.js',
+    filename: 'bundle.js',
     path: path.resolve(__dirname, 'dist'),
-    libraryTarget: 'commonjs2'
+    clean: true
   },
   resolve: {
-    extensions: ['.ts', '.js']
+    extensions: ['.tsx', '.ts', '.js', '.jsx']
   },
   module: {
     rules: [
       {
-        test: /\.ts$/,
+        test: /\.tsx?$/,
         use: 'ts-loader',
         exclude: /node_modules/
+      },
+      {
+        test: /\.css$/,
+        use: ['style-loader', 'css-loader']
       }
     ]
   },
-  externals: {
-    // Add any externals here if needed
-    // For example, if you want to exclude flowquery from the bundle:
-    // 'flowquery': 'commonjs flowquery'
+  plugins: [
+    new HtmlWebpackPlugin({
+      template: './public/index.html'
+    }),
+  ],
+  devServer: {
+    static: './dist',
+    port: 3000,
+    open: true,
+    hot: true
   },
   optimization: {
-    minimize: true
+    minimize: false
   }
 };

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "flowquery",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",
+  "sideEffects": true,
   "exports": {
     ".": {
       "import": "./dist/index.node.js",
@@ -13,6 +14,11 @@
     "./browser": {
       "import": "./dist/flowquery.min.js",
       "require": "./dist/flowquery.min.js"
+    },
+    "./extensibility": {
+      "import": "./dist/extensibility.js",
+      "require": "./dist/extensibility.js",
+      "types": "./dist/extensibility.d.ts"
     }
   },
   "bin": {

package/src/extensibility.ts

@@ -0,0 +1,9 @@
+/**
+ * FlowQuery Extensibility API
+ * 
+ * This module re-exports the extensibility API from the parsing/functions/extensibility module.
+ * 
+ * @packageDocumentation
+ */
+
+export * from "./parsing/functions/extensibility";

package/src/parsing/functions/avg.ts

@@ -1,6 +1,7 @@
 import AggregateFunction from "./aggregate_function";
 import ASTNode from "../ast_node";
 import ReducerElement from "./reducer_element";
+import { FunctionDef } from "./function_metadata";
 
 class AvgReducerElement extends ReducerElement {
     private _count: number = 0;
@@ -21,6 +22,15 @@ class AvgReducerElement extends ReducerElement {
     }
 }
 
+@FunctionDef({
+    description: "Calculates the average of numeric values across grouped rows",
+    category: "aggregate",
+    parameters: [
+        { name: "value", description: "Numeric value to average", type: "number" }
+    ],
+    output: { description: "Average of all values", type: "number", example: 50 },
+    examples: ["WITH [10, 20, 30] AS nums UNWIND nums AS n RETURN avg(n)"]
+})
 class Avg extends AggregateFunction {
     constructor() {
         super("avg");

package/src/parsing/functions/collect.ts

@@ -1,5 +1,6 @@
 import AggregateFunction from "./aggregate_function";
 import ReducerElement from "./reducer_element";
+import { FunctionDef } from "./function_metadata";
 
 class CollectReducerElement extends ReducerElement {
     private _value: any[] = [];
@@ -24,6 +25,15 @@ class DistinctCollectReducerElement extends ReducerElement {
     }
 }
 
+@FunctionDef({
+    description: "Collects values into an array across grouped rows",
+    category: "aggregate",
+    parameters: [
+        { name: "value", description: "Value to collect", type: "any" }
+    ],
+    output: { description: "Array of collected values", type: "array", example: [1, 2, 3] },
+    examples: ["WITH [1, 2, 3] AS nums UNWIND nums AS n RETURN collect(n)"]
+})
 class Collect extends AggregateFunction {
     private _distinct: boolean = false;
     constructor() {

package/src/parsing/functions/extensibility/index.ts

@@ -0,0 +1,54 @@
+/**
+ * FlowQuery Extensibility API
+ * 
+ * This module provides all the exports needed to create custom FlowQuery functions.
+ * 
+ * @packageDocumentation
+ * 
+ * @example
+ * ```typescript
+ * import { Function, FunctionDef } from '../parsing/functions/extensibility';
+ * 
+ * @FunctionDef({
+ *     description: "Converts a string to uppercase",
+ *     category: "string",
+ *     parameters: [{ name: "text", description: "String to convert", type: "string" }],
+ *     output: { description: "Uppercase string", type: "string" }
+ * })
+ * class UpperCase extends Function {
+ *     constructor() {
+ *         super("uppercase");
+ *         this._expectedParameterCount = 1;
+ *     }
+ *     public value(): string {
+ *         return String(this.getChildren()[0].value()).toUpperCase();
+ *     }
+ * }
+ * ```
+ */
+
+// Base function classes for creating custom functions
+export { default as Function } from "../function";
+export { default as AggregateFunction } from "../aggregate_function";
+export { default as PredicateFunction } from "../predicate_function";
+export { default as ReducerElement } from "../reducer_element";
+
+// Decorator and metadata types for function registration
+export { 
+    FunctionDef,
+    FunctionMetadata,
+    FunctionDefOptions,
+    ParameterSchema,
+    OutputSchema,
+    FunctionCategory
+} from "../function_metadata";
+
+// Factory for advanced usage
+export { default as FunctionFactory } from "../function_factory";
+export type { FunctionCreator, AsyncDataProvider } from "../function_factory";
+
+// Registration option types
+export type {
+    RegisterFunctionOptions,
+    RegisterAsyncProviderOptions
+} from "../function_metadata";

package/src/parsing/functions/function_factory.ts

@@ -1,22 +1,27 @@
-import Sum from "./sum";
-import Collect from "./collect";
-import Avg from "./avg";
-import Range from "./range";
-import Rand from "./rand";
-import Round from "./round";
-import Split from "./split";
-import Join from "./join";
-import ToJson from "./to_json";
-import Replace from "./replace";
-import Stringify from "./stringify";
-import Size from "./size";
-import Functions from "./functions";
 import Function from "./function";
+import PredicateFunction from "./predicate_function";
+// Import built-in functions to ensure their @FunctionDef decorators run
+import "./sum";
+import "./collect";
+import "./avg";
+import "./range";
+import "./rand";
+import "./round";
+import "./split";
+import "./join";
+import "./to_json";
+import "./replace";
+import "./stringify";
+import "./size";
+import "./functions";
+import "./predicate_sum";
 import { 
     FunctionMetadata, 
     RegisterFunctionOptions, 
     RegisterAsyncProviderOptions,
-    BUILTIN_FUNCTION_METADATA 
+    getRegisteredFunctionMetadata,
+    getFunctionMetadata,
+    getRegisteredFunctionFactory
 } from "./function_metadata";
 
 /**
@@ -182,8 +187,8 @@ class FunctionFactory {
         if (FunctionFactory.metadata.has(lowerName)) {
             return FunctionFactory.metadata.get(lowerName);
         }
-        // Fall back to built-in metadata
-        return BUILTIN_FUNCTION_METADATA.find(m => m.name === lowerName);
+        // Fall back to decorator-registered metadata
+        return getFunctionMetadata(lowerName);
     }
 
     /**
@@ -202,9 +207,9 @@ class FunctionFactory {
         const result: FunctionMetadata[] = [];
         const includeBuiltins = options?.includeBuiltins !== false;
         
-        // Add built-in functions
+        // Add decorator-registered functions (built-ins)
         if (includeBuiltins) {
-            for (const meta of BUILTIN_FUNCTION_METADATA) {
+            for (const meta of getRegisteredFunctionMetadata()) {
                 if (options?.category && meta.category !== options.category) continue;
                 if (options?.asyncOnly) continue; // Built-ins are sync
                 result.push(meta);
@@ -228,7 +233,7 @@ class FunctionFactory {
      * @returns Array of function names
      */
     public static listFunctionNames(): string[] {
-        const builtinNames = BUILTIN_FUNCTION_METADATA.map(m => m.name);
+        const builtinNames = getRegisteredFunctionMetadata().map(m => m.name);
         const pluginNames = Array.from(FunctionFactory.plugins.keys());
         const asyncNames = Array.from(FunctionFactory.asyncProviders.keys());
         return [...new Set([...builtinNames, ...pluginNames, ...asyncNames])];
@@ -259,36 +264,34 @@ class FunctionFactory {
             return FunctionFactory.plugins.get(lowerName)!();
         }
 
-        switch (lowerName) {
-            case "sum":
-                return new Sum();
-            case "collect":
-                return new Collect();
-            case "avg":
-                return new Avg();
-            case "range":
-                return new Range();
-            case "rand":
-                return new Rand();
-            case "round":
-                return new Round();
-            case "split":
-                return new Split();
-            case "join":
-                return new Join();
-            case "tojson":
-                return new ToJson();
-            case "replace":
-                return new Replace();
-            case "stringify":
-                return new Stringify();
-            case "size":
-                return new Size();
-            case "functions":
-                return new Functions();
-            default:
-                return new Function(name);
+        // Check decorator-registered functions (built-ins use @FunctionDef)
+        const decoratorFactory = getRegisteredFunctionFactory(lowerName);
+        if (decoratorFactory) {
+            return decoratorFactory();
         }
+
+        // Unknown function - return generic Function instance
+        return new Function(name);
+    }
+
+    /**
+     * Creates a predicate function instance by name.
+     * Predicate functions are used in WHERE clauses with quantifiers (e.g., ANY, ALL).
+     * 
+     * @param name - The function name (case-insensitive)
+     * @returns A PredicateFunction instance of the appropriate type
+     */
+    public static createPredicate(name: string): PredicateFunction {
+        const lowerName = name.toLowerCase();
+        
+        // Check decorator-registered predicate functions
+        const decoratorFactory = getRegisteredFunctionFactory(lowerName, 'predicate');
+        if (decoratorFactory) {
+            return decoratorFactory();
+        }
+
+        // Unknown predicate function - return generic PredicateFunction instance
+        return new PredicateFunction(name);
     }
 }