flowquery 1.0.5 → 1.0.7

package/misc/apps/RAG/src/utils/FlowQueryExtractor.ts

@@ -0,0 +1,203 @@
+/**
+ * FlowQuery Extraction Utility
+ * 
+ * Extracts FlowQuery statements from LLM responses.
+ */
+
+/**
+ * Extraction result containing the query and any explanation text.
+ */
+export interface FlowQueryExtraction {
+    /** The extracted FlowQuery statement, if found */
+    query: string | null;
+    /** Whether a query was successfully extracted */
+    found: boolean;
+    /** Any explanation text before the code block */
+    explanation?: string;
+    /** Whether the LLM indicated no query is needed */
+    noQueryNeeded?: boolean;
+    /** Message from the LLM if no query is needed */
+    directResponse?: string;
+}
+
+/**
+ * FlowQuery Extractor class for extracting FlowQuery statements from LLM responses.
+ * 
+ * Looks for code blocks with flowquery, cypher, or sql language tags,
+ * or generic code blocks that appear to contain FlowQuery syntax.
+ * 
+ * @example
+ * ```typescript
+ * const extractor = new FlowQueryExtractor();
+ * const response = `Here's the query:
+ * \`\`\`flowquery
+ * LOAD JSON FROM somePlugin(5) AS item
+ * RETURN item.text
+ * \`\`\``;
+ * 
+ * const extraction = extractor.extract(response);
+ * console.log(extraction.query); // "LOAD JSON FROM somePlugin(5) AS item\nRETURN item.text"
+ * ```
+ */
+export class FlowQueryExtractor {
+    /** Regex patterns for matching code blocks with language tags */
+    private readonly codeBlockPatterns: RegExp[] = [
+        /```(?:flowquery|cypher|fql)\s*\n([\s\S]*?)```/i,
+        /```(?:sql)\s*\n([\s\S]*?)```/i,
+        /```\s*\n([\s\S]*?)```/,  // Generic code block
+    ];
+
+    /** Keywords that indicate a FlowQuery statement */
+    private readonly flowQueryKeywords: RegExp[] = [
+        /\bWITH\b/i,
+        /\bLOAD\s+JSON\s+FROM\b/i,
+        /\bUNWIND\b/i,
+        /\bRETURN\b/i,
+        /\bWHERE\b/i,
+        /\bORDER\s+BY\b/i,
+        /\bLIMIT\b/i,
+    ];
+
+    /** Keywords that can start a FlowQuery statement */
+    private readonly startKeywords = /^(WITH|LOAD\s+JSON\s+FROM|UNWIND)\b/i;
+
+    /** Keywords that can continue a FlowQuery statement */
+    private readonly continueKeywords = /^(WITH|LOAD|UNWIND|WHERE|RETURN|ORDER|LIMIT|SKIP|HEADERS|POST|AS)\b/i;
+
+    /**
+     * Extract a FlowQuery statement from an LLM response.
+     * 
+     * @param llmResponse - The full text response from the LLM
+     * @returns The extraction result
+     */
+    public extract(llmResponse: string): FlowQueryExtraction {
+        if (!llmResponse || llmResponse.trim() === '') {
+            return { query: null, found: false };
+        }
+
+        // Check for explicit "NO_QUERY_NEEDED" marker
+        if (llmResponse.includes('[NO_QUERY_NEEDED]') || 
+            llmResponse.includes('NO_QUERY_NEEDED')) {
+            // Extract the direct response after the marker
+            const directMatch = llmResponse.match(/\[NO_QUERY_NEEDED\]\s*([\s\S]*)/i);
+            return {
+                query: null,
+                found: false,
+                noQueryNeeded: true,
+                directResponse: directMatch ? directMatch[1].trim() : llmResponse
+            };
+        }
+
+        // Try to match code blocks with specific language tags
+        for (const pattern of this.codeBlockPatterns) {
+            const match = llmResponse.match(pattern);
+            if (match && match[1]) {
+                const query = match[1].trim();
+                
+                // Verify it looks like a FlowQuery statement
+                if (this.isLikelyFlowQuery(query)) {
+                    // Extract explanation text before the code block
+                    const beforeMatch = llmResponse.substring(0, llmResponse.indexOf(match[0])).trim();
+                    
+                    return {
+                        query,
+                        found: true,
+                        explanation: beforeMatch || undefined
+                    };
+                }
+            }
+        }
+
+        // Try to extract inline FlowQuery if no code block found
+        // Look for lines starting with FlowQuery keywords
+        const inlineQuery = this.extractInlineQuery(llmResponse);
+        if (inlineQuery) {
+            return {
+                query: inlineQuery,
+                found: true
+            };
+        }
+
+        return { query: null, found: false };
+    }
+
+    /**
+     * Extract multiple FlowQuery statements from an LLM response.
+     * Useful when the LLM provides alternative queries.
+     * 
+     * @param llmResponse - The full text response from the LLM
+     * @returns Array of extracted queries
+     */
+    public extractAll(llmResponse: string): string[] {
+        if (!llmResponse) return [];
+
+        const queries: string[] = [];
+        const codeBlockPattern = /```(?:flowquery|cypher|fql|sql)?\s*\n([\s\S]*?)```/gi;
+        
+        let match;
+        while ((match = codeBlockPattern.exec(llmResponse)) !== null) {
+            if (match[1]) {
+                const query = match[1].trim();
+                if (this.isLikelyFlowQuery(query)) {
+                    queries.push(query);
+                }
+            }
+        }
+
+        return queries;
+    }
+
+    /**
+     * Check if a string looks like a FlowQuery statement.
+     */
+    private isLikelyFlowQuery(text: string): boolean {
+        // Must contain at least one FlowQuery keyword
+        return this.flowQueryKeywords.some(pattern => pattern.test(text));
+    }
+
+    /**
+     * Try to extract a FlowQuery statement that's not in a code block.
+     */
+    private extractInlineQuery(text: string): string | null {
+        const lines = text.split('\n');
+        const queryLines: string[] = [];
+        let inQuery = false;
+
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            
+            if (!inQuery && this.startKeywords.test(trimmedLine)) {
+                inQuery = true;
+                queryLines.push(trimmedLine);
+            } else if (inQuery) {
+                // Check if this line continues the query
+                if (this.continueKeywords.test(trimmedLine) || 
+                    trimmedLine.startsWith('{') ||
+                    trimmedLine.startsWith('}') ||
+                    trimmedLine === '' ||
+                    /^[A-Za-z_][A-Za-z0-9_]*\./.test(trimmedLine) ||
+                    /^\s+/.test(line)) {  // Indented line
+                    queryLines.push(trimmedLine);
+                } else {
+                    // End of query
+                    break;
+                }
+            }
+        }
+
+        const query = queryLines.join('\n').trim();
+        return query && this.isLikelyFlowQuery(query) ? query : null;
+    }
+}
+
+// Convenience function for backward compatibility
+export function extractFlowQuery(llmResponse: string): FlowQueryExtraction {
+    return new FlowQueryExtractor().extract(llmResponse);
+}
+
+// Convenience function for backward compatibility
+export function extractAllFlowQueries(llmResponse: string): string[] {
+    return new FlowQueryExtractor().extractAll(llmResponse);
+}
+
+export default FlowQueryExtractor;

package/misc/apps/RAG/src/utils/index.ts

@@ -0,0 +1,9 @@
+/**
+ * Utils module exports
+ */
+
+export { FlowQueryExecutor } from './FlowQueryExecutor';
+export type { FlowQueryExecutionResult } from './FlowQueryExecutor';
+
+export { FlowQueryExtractor, extractFlowQuery, extractAllFlowQueries } from './FlowQueryExtractor';
+export type { FlowQueryExtraction } from './FlowQueryExtractor';

package/misc/apps/RAG/tsconfig.json

@@ -3,8 +3,8 @@
     "outDir": "./dist",
     "rootDir": "./src",
     "target": "ES2020",
-    "module": "commonjs",
-    "moduleResolution": "node",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "strict": true,
@@ -12,6 +12,8 @@
     "declaration": true,
     "sourceMap": true,
     "resolveJsonModule": true,
+    "jsx": "react-jsx",
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
     "types": ["node"]
   },
   "include": ["src/**/*"],

package/misc/apps/RAG/webpack.config.js

@@ -1,32 +1,43 @@
 const path = require('path');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
 
 module.exports = {
-  mode: 'production',
-  entry: './src/index.ts',
-  target: 'node',
+  mode: 'development',
+  entry: './src/index.tsx',
+  target: 'web',
   output: {
-    filename: 'rag.bundle.js',
+    filename: 'bundle.js',
     path: path.resolve(__dirname, 'dist'),
-    libraryTarget: 'commonjs2'
+    clean: true
   },
   resolve: {
-    extensions: ['.ts', '.js']
+    extensions: ['.tsx', '.ts', '.js', '.jsx']
   },
   module: {
     rules: [
       {
-        test: /\.ts$/,
+        test: /\.tsx?$/,
         use: 'ts-loader',
         exclude: /node_modules/
+      },
+      {
+        test: /\.css$/,
+        use: ['style-loader', 'css-loader']
       }
     ]
   },
-  externals: {
-    // Add any externals here if needed
-    // For example, if you want to exclude flowquery from the bundle:
-    // 'flowquery': 'commonjs flowquery'
+  plugins: [
+    new HtmlWebpackPlugin({
+      template: './public/index.html'
+    }),
+  ],
+  devServer: {
+    static: './dist',
+    port: 3000,
+    open: true,
+    hot: true
   },
   optimization: {
-    minimize: true
+    minimize: false
   }
 };

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "flowquery",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",
+  "sideEffects": true,
   "exports": {
     ".": {
       "import": "./dist/index.node.js",
@@ -13,6 +14,11 @@
     "./browser": {
       "import": "./dist/flowquery.min.js",
       "require": "./dist/flowquery.min.js"
+    },
+    "./extensibility": {
+      "import": "./dist/extensibility.js",
+      "require": "./dist/extensibility.js",
+      "types": "./dist/extensibility.d.ts"
     }
   },
   "bin": {

package/src/compute/runner.ts

@@ -1,7 +1,6 @@
 import Operation from "../parsing/operations/operation";
 import Parser from "../parsing/parser";
-import { FunctionCreator, AsyncDataProvider } from "../parsing/functions/function_factory";
-import { FunctionMetadata, RegisterFunctionOptions, RegisterAsyncProviderOptions } from "../parsing/functions/function_metadata";
+import { FunctionMetadata } from "../parsing/functions/function_metadata";
 import Function from "../parsing/functions/function";
 
 /**
@@ -21,30 +20,6 @@ class Runner {
     private first: Operation;
     private last: Operation;
 
-    /**
-     * Register a synchronous plugin function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static registerFunction: (name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions) => void;
-
-    /**
-     * Unregister a synchronous plugin function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static unregisterFunction: (name: string) => void;
-
-    /**
-     * Register an async data provider function for use in LOAD operations.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static registerAsyncProvider: (name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions) => void;
-
-    /**
-     * Unregister an async data provider function.
-     * Added dynamically in index.browser.ts / index.node.ts
-     */
-    static unregisterAsyncProvider: (name: string) => void;
-
     /**
      * List all registered functions with their metadata.
      * Added dynamically in index.browser.ts / index.node.ts

package/src/extensibility.ts

@@ -0,0 +1,45 @@
+/**
+ * FlowQuery Extensibility API
+ * 
+ * This module provides all the exports needed to create custom FlowQuery functions.
+ * 
+ * @packageDocumentation
+ * 
+ * @example
+ * ```typescript
+ * import { Function, FunctionDef } from 'flowquery/extensibility';
+ * 
+ * @FunctionDef({
+ *     description: "Converts a string to uppercase",
+ *     category: "string",
+ *     parameters: [{ name: "text", description: "String to convert", type: "string" }],
+ *     output: { description: "Uppercase string", type: "string" }
+ * })
+ * class UpperCase extends Function {
+ *     constructor() {
+ *         super("uppercase");
+ *         this._expectedParameterCount = 1;
+ *     }
+ *     public value(): string {
+ *         return String(this.getChildren()[0].value()).toUpperCase();
+ *     }
+ * }
+ * ```
+ */
+
+// Base function classes for creating custom functions
+export { default as Function } from "./parsing/functions/function";
+export { default as AggregateFunction } from "./parsing/functions/aggregate_function";
+export { default as AsyncFunction } from "./parsing/functions/async_function";
+export { default as PredicateFunction } from "./parsing/functions/predicate_function";
+export { default as ReducerElement } from "./parsing/functions/reducer_element";
+
+// Decorator and metadata types for function registration
+export { 
+    FunctionDef,
+    FunctionMetadata,
+    FunctionDefOptions,
+    ParameterSchema,
+    OutputSchema,
+    FunctionCategory
+} from "./parsing/functions/function_metadata";

package/src/index.browser.ts

@@ -7,100 +7,14 @@
  */
 
 import {default as FlowQuery} from "./compute/runner";
-import FunctionFactory, { FunctionCreator, AsyncDataProvider } from "./parsing/functions/function_factory";
+import FunctionFactory from "./parsing/functions/function_factory";
 import { 
     FunctionMetadata, 
     ParameterSchema, 
-    OutputSchema,
-    RegisterFunctionOptions,
-    RegisterAsyncProviderOptions 
+    OutputSchema
 } from "./parsing/functions/function_metadata";
 import Function from "./parsing/functions/function";
 
-/**
- * Register a synchronous plugin function.
- * 
- * @param name - The function name
- * @param factoryOrOptions - Factory function or options object with metadata
- * 
- * @example
- * ```javascript
- * // Simple registration
- * FlowQuery.registerFunction("uppercase", () => new MyUpperCaseFunction());
- * 
- * // Registration with metadata for LLM consumption
- * FlowQuery.registerFunction("uppercase", {
- *     factory: () => new MyUpperCaseFunction(),
- *     metadata: {
- *         name: "uppercase",
- *         description: "Converts a string to uppercase",
- *         category: "string",
- *         parameters: [{ name: "text", description: "String to convert", type: "string" }],
- *         output: { description: "Uppercase string", type: "string" },
- *         examples: ["WITH 'hello' AS s RETURN uppercase(s)"]
- *     }
- * });
- * ```
- */
-FlowQuery.registerFunction = function(name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions): void {
-    FunctionFactory.register(name, factoryOrOptions);
-};
-
-/**
- * Unregister a synchronous plugin function.
- * 
- * @param name - The function name
- */
-FlowQuery.unregisterFunction = function(name: string): void {
-    FunctionFactory.unregister(name);
-};
-
-/**
- * Register an async data provider function for use in LOAD operations.
- * 
- * @param name - The function name
- * @param providerOrOptions - Async provider or options object with metadata
- * 
- * @example
- * ```javascript
- * // Registration with metadata for LLM consumption
- * FlowQuery.registerAsyncProvider("fetchUsers", {
- *     provider: async function* (endpoint) {
- *         const response = await fetch(endpoint);
- *         const data = await response.json();
- *         for (const item of data) yield item;
- *     },
- *     metadata: {
- *         name: "fetchUsers",
- *         description: "Fetches user data from an API",
- *         category: "data",
- *         parameters: [{ name: "endpoint", description: "API URL", type: "string" }],
- *         output: { 
- *             description: "User object", 
- *             type: "object",
- *             properties: {
- *                 id: { description: "User ID", type: "number" },
- *                 name: { description: "User name", type: "string" }
- *             }
- *         },
- *         examples: ["LOAD JSON FROM fetchUsers('https://api.example.com/users') AS user"]
- *     }
- * });
- * ```
- */
-FlowQuery.registerAsyncProvider = function(name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions): void {
-    FunctionFactory.registerAsyncProvider(name, providerOrOptions);
-};
-
-/**
- * Unregister an async data provider function.
- * 
- * @param name - The function name
- */
-FlowQuery.unregisterAsyncProvider = function(name: string): void {
-    FunctionFactory.unregisterAsyncProvider(name);
-};
-
 /**
  * List all registered functions with their metadata.
  * 

package/src/index.node.ts

@@ -7,100 +7,14 @@
  */
 
 import {default as FlowQuery} from "./compute/runner";
-import FunctionFactory, { FunctionCreator, AsyncDataProvider } from "./parsing/functions/function_factory";
+import FunctionFactory, { AsyncDataProvider } from "./parsing/functions/function_factory";
 import { 
     FunctionMetadata, 
     ParameterSchema, 
-    OutputSchema,
-    RegisterFunctionOptions,
-    RegisterAsyncProviderOptions 
+    OutputSchema
 } from "./parsing/functions/function_metadata";
 import Function from "./parsing/functions/function";
 
-/**
- * Register a synchronous plugin function.
- * 
- * @param name - The function name
- * @param factoryOrOptions - Factory function or options object with metadata
- * 
- * @example
- * ```javascript
- * // Simple registration
- * FlowQuery.registerFunction("uppercase", () => new MyUpperCaseFunction());
- * 
- * // Registration with metadata for LLM consumption
- * FlowQuery.registerFunction("uppercase", {
- *     factory: () => new MyUpperCaseFunction(),
- *     metadata: {
- *         name: "uppercase",
- *         description: "Converts a string to uppercase",
- *         category: "string",
- *         parameters: [{ name: "text", description: "String to convert", type: "string" }],
- *         output: { description: "Uppercase string", type: "string" },
- *         examples: ["WITH 'hello' AS s RETURN uppercase(s)"]
- *     }
- * });
- * ```
- */
-FlowQuery.registerFunction = function(name: string, factoryOrOptions: FunctionCreator | RegisterFunctionOptions): void {
-    FunctionFactory.register(name, factoryOrOptions);
-};
-
-/**
- * Unregister a synchronous plugin function.
- * 
- * @param name - The function name
- */
-FlowQuery.unregisterFunction = function(name: string): void {
-    FunctionFactory.unregister(name);
-};
-
-/**
- * Register an async data provider function for use in LOAD operations.
- * 
- * @param name - The function name
- * @param providerOrOptions - Async provider or options object with metadata
- * 
- * @example
- * ```javascript
- * // Registration with metadata for LLM consumption
- * FlowQuery.registerAsyncProvider("fetchUsers", {
- *     provider: async function* (endpoint) {
- *         const response = await fetch(endpoint);
- *         const data = await response.json();
- *         for (const item of data) yield item;
- *     },
- *     metadata: {
- *         name: "fetchUsers",
- *         description: "Fetches user data from an API",
- *         category: "data",
- *         parameters: [{ name: "endpoint", description: "API URL", type: "string" }],
- *         output: { 
- *             description: "User object", 
- *             type: "object",
- *             properties: {
- *                 id: { description: "User ID", type: "number" },
- *                 name: { description: "User name", type: "string" }
- *             }
- *         },
- *         examples: ["LOAD JSON FROM fetchUsers('https://api.example.com/users') AS user"]
- *     }
- * });
- * ```
- */
-FlowQuery.registerAsyncProvider = function(name: string, providerOrOptions: AsyncDataProvider | RegisterAsyncProviderOptions): void {
-    FunctionFactory.registerAsyncProvider(name, providerOrOptions);
-};
-
-/**
- * Unregister an async data provider function.
- * 
- * @param name - The function name
- */
-FlowQuery.unregisterAsyncProvider = function(name: string): void {
-    FunctionFactory.unregisterAsyncProvider(name);
-};
-
 /**
  * List all registered functions with their metadata.
  * 
@@ -131,11 +45,8 @@ export {
     FlowQuery, 
     Function, 
     FunctionFactory, 
-    FunctionCreator, 
     AsyncDataProvider,
     FunctionMetadata,
     ParameterSchema,
-    OutputSchema,
-    RegisterFunctionOptions,
-    RegisterAsyncProviderOptions
+    OutputSchema
 };

package/src/parsing/functions/avg.ts

@@ -1,6 +1,7 @@
 import AggregateFunction from "./aggregate_function";
 import ASTNode from "../ast_node";
 import ReducerElement from "./reducer_element";
+import { FunctionDef } from "./function_metadata";
 
 class AvgReducerElement extends ReducerElement {
     private _count: number = 0;
@@ -21,6 +22,15 @@ class AvgReducerElement extends ReducerElement {
     }
 }
 
+@FunctionDef({
+    description: "Calculates the average of numeric values across grouped rows",
+    category: "aggregate",
+    parameters: [
+        { name: "value", description: "Numeric value to average", type: "number" }
+    ],
+    output: { description: "Average of all values", type: "number", example: 50 },
+    examples: ["WITH [10, 20, 30] AS nums UNWIND nums AS n RETURN avg(n)"]
+})
 class Avg extends AggregateFunction {
     constructor() {
         super("avg");

package/src/parsing/functions/collect.ts

@@ -1,5 +1,6 @@
 import AggregateFunction from "./aggregate_function";
 import ReducerElement from "./reducer_element";
+import { FunctionDef } from "./function_metadata";
 
 class CollectReducerElement extends ReducerElement {
     private _value: any[] = [];
@@ -24,6 +25,15 @@ class DistinctCollectReducerElement extends ReducerElement {
     }
 }
 
+@FunctionDef({
+    description: "Collects values into an array across grouped rows",
+    category: "aggregate",
+    parameters: [
+        { name: "value", description: "Value to collect", type: "any" }
+    ],
+    output: { description: "Array of collected values", type: "array", example: [1, 2, 3] },
+    examples: ["WITH [1, 2, 3] AS nums UNWIND nums AS n RETURN collect(n)"]
+})
 class Collect extends AggregateFunction {
     private _distinct: boolean = false;
     constructor() {