flowquery 1.0.25 → 1.0.26

package/misc/apps/RAG/src/utils/FlowQueryExecutor.ts

@@ -1,10 +1,9 @@
 /**
  * FlowQuery Executor Utility
- * 
+ *
  * Executes FlowQuery statements and handles errors gracefully.
  */
-
-import { FlowQuery } from 'flowquery';
+import { FlowQuery } from "flowquery";
 
 /**
  * Result of executing a FlowQuery statement.
@@ -24,7 +23,7 @@ export interface FlowQueryExecutionResult {
 
 /**
  * FlowQuery Executor class for executing FlowQuery statements.
- * 
+ *
  * @example
  * ```typescript
  * const executor = new FlowQueryExecutor();
@@ -49,52 +48,52 @@ export class FlowQueryExecutor {
 
     /**
      * Execute a FlowQuery statement and return the results.
-     * 
+     *
      * @param query - The FlowQuery statement to execute
      * @returns The execution result including success status, results or error
      */
     async execute(query: string): Promise<FlowQueryExecutionResult> {
         const startTime = performance.now();
-        
+
         try {
             // Validate the query is not empty
-            if (!query || query.trim() === '') {
+            if (!query || query.trim() === "") {
                 return {
                     success: false,
                     query,
-                    error: 'Query cannot be empty',
-                    executionTime: performance.now() - startTime
+                    error: "Query cannot be empty",
+                    executionTime: performance.now() - startTime,
                 };
             }
 
             // Create a runner and execute the query
             const runner = new FlowQuery(query);
             await runner.run();
-            
+
             // Get the results
             const results = runner.results;
-            
+
             return {
                 success: true,
                 query,
                 results: Array.isArray(results) ? results : [results],
-                executionTime: performance.now() - startTime
+                executionTime: performance.now() - startTime,
             };
         } catch (error) {
             const errorMessage = error instanceof Error ? error.message : String(error);
-            
+
             return {
                 success: false,
                 query,
                 error: errorMessage,
-                executionTime: performance.now() - startTime
+                executionTime: performance.now() - startTime,
             };
         }
     }
 
     /**
      * Format execution results for display or LLM consumption.
-     * 
+     *
      * @param result - The execution result to format
      * @param maxItems - Maximum number of items to include (uses default if not specified)
      * @returns A formatted string representation of the results
@@ -109,21 +108,21 @@ export class FlowQueryExecutor {
         const results = result.results || [];
         const totalCount = results.length;
         const displayResults = results.slice(0, limit);
-        
+
         let output = `Execution successful (${result.executionTime.toFixed(2)}ms)\n`;
         output += `Total results: ${totalCount}\n\n`;
-        
+
         if (totalCount === 0) {
-            output += 'No results returned.';
+            output += "No results returned.";
         } else {
-            output += 'Results:\n';
+            output += "Results:\n";
             output += JSON.stringify(displayResults, null, 2);
-            
+
             if (totalCount > limit) {
                 output += `\n\n... and ${totalCount - limit} more results`;
             }
         }
-        
+
         return output;
     }
 }

package/misc/apps/RAG/src/utils/FlowQueryExtractor.ts

@@ -1,6 +1,6 @@
 /**
  * FlowQuery Extraction Utility
- * 
+ *
  * Extracts FlowQuery statements from LLM responses.
  */
 
@@ -22,10 +22,10 @@ export interface FlowQueryExtraction {
 
 /**
  * FlowQuery Extractor class for extracting FlowQuery statements from LLM responses.
- * 
+ *
  * Looks for code blocks with flowquery, cypher, or sql language tags,
  * or generic code blocks that appear to contain FlowQuery syntax.
- * 
+ *
  * @example
  * ```typescript
  * const extractor = new FlowQueryExtractor();
@@ -34,7 +34,7 @@ export interface FlowQueryExtraction {
  * LOAD JSON FROM somePlugin(5) AS item
  * RETURN item.text
  * \`\`\``;
- * 
+ *
  * const extraction = extractor.extract(response);
  * console.log(extraction.query); // "LOAD JSON FROM somePlugin(5) AS item\nRETURN item.text"
  * ```
@@ -44,47 +44,47 @@ export class FlowQueryExtractor {
     private readonly codeBlockPatterns: RegExp[] = [
         /```(?:flowquery|cypher|fql)\s*\n([\s\S]*?)```/i,
         /```(?:sql)\s*\n([\s\S]*?)```/i,
-        /```\s*\n([\s\S]*?)```/,  // Generic code block
+        /```\s*\n([\s\S]*?)```/, // Generic code block
     ];
 
     /** Keywords that indicate a FlowQuery statement */
     private readonly flowQueryKeywords: RegExp[] = [
+        /\bMATCH\b/i,
+        /\bCREATE\b/i,
         /\bWITH\b/i,
         /\bLOAD\s+JSON\s+FROM\b/i,
         /\bUNWIND\b/i,
         /\bRETURN\b/i,
         /\bWHERE\b/i,
-        /\bORDER\s+BY\b/i,
-        /\bLIMIT\b/i,
     ];
 
     /** Keywords that can start a FlowQuery statement */
-    private readonly startKeywords = /^(WITH|LOAD\s+JSON\s+FROM|UNWIND)\b/i;
+    private readonly startKeywords = /^(MATCH|CREATE|WITH|LOAD\s+JSON\s+FROM|UNWIND|CALL)\b/i;
 
     /** Keywords that can continue a FlowQuery statement */
-    private readonly continueKeywords = /^(WITH|LOAD|UNWIND|WHERE|RETURN|ORDER|LIMIT|SKIP|HEADERS|POST|AS)\b/i;
+    private readonly continueKeywords =
+        /^(MATCH|CREATE|WITH|LOAD|UNWIND|WHERE|RETURN|CALL|YIELD|HEADERS|POST|AS)\b/i;
 
     /**
      * Extract a FlowQuery statement from an LLM response.
-     * 
+     *
      * @param llmResponse - The full text response from the LLM
      * @returns The extraction result
      */
     public extract(llmResponse: string): FlowQueryExtraction {
-        if (!llmResponse || llmResponse.trim() === '') {
+        if (!llmResponse || llmResponse.trim() === "") {
             return { query: null, found: false };
         }
 
         // Check for explicit "NO_QUERY_NEEDED" marker
-        if (llmResponse.includes('[NO_QUERY_NEEDED]') || 
-            llmResponse.includes('NO_QUERY_NEEDED')) {
+        if (llmResponse.includes("[NO_QUERY_NEEDED]") || llmResponse.includes("NO_QUERY_NEEDED")) {
             // Extract the direct response after the marker
             const directMatch = llmResponse.match(/\[NO_QUERY_NEEDED\]\s*([\s\S]*)/i);
             return {
                 query: null,
                 found: false,
                 noQueryNeeded: true,
-                directResponse: directMatch ? directMatch[1].trim() : llmResponse
+                directResponse: directMatch ? directMatch[1].trim() : llmResponse,
             };
         }
 
@@ -93,16 +93,18 @@ export class FlowQueryExtractor {
             const match = llmResponse.match(pattern);
             if (match && match[1]) {
                 const query = match[1].trim();
-                
+
                 // Verify it looks like a FlowQuery statement
                 if (this.isLikelyFlowQuery(query)) {
                     // Extract explanation text before the code block
-                    const beforeMatch = llmResponse.substring(0, llmResponse.indexOf(match[0])).trim();
-                    
+                    const beforeMatch = llmResponse
+                        .substring(0, llmResponse.indexOf(match[0]))
+                        .trim();
+
                     return {
                         query,
                         found: true,
-                        explanation: beforeMatch || undefined
+                        explanation: beforeMatch || undefined,
                     };
                 }
             }
@@ -114,7 +116,7 @@ export class FlowQueryExtractor {
         if (inlineQuery) {
             return {
                 query: inlineQuery,
-                found: true
+                found: true,
             };
         }
 
@@ -124,7 +126,7 @@ export class FlowQueryExtractor {
     /**
      * Extract multiple FlowQuery statements from an LLM response.
      * Useful when the LLM provides alternative queries.
-     * 
+     *
      * @param llmResponse - The full text response from the LLM
      * @returns Array of extracted queries
      */
@@ -133,7 +135,7 @@ export class FlowQueryExtractor {
 
         const queries: string[] = [];
         const codeBlockPattern = /```(?:flowquery|cypher|fql|sql)?\s*\n([\s\S]*?)```/gi;
-        
+
         let match;
         while ((match = codeBlockPattern.exec(llmResponse)) !== null) {
             if (match[1]) {
@@ -152,31 +154,34 @@ export class FlowQueryExtractor {
      */
     private isLikelyFlowQuery(text: string): boolean {
         // Must contain at least one FlowQuery keyword
-        return this.flowQueryKeywords.some(pattern => pattern.test(text));
+        return this.flowQueryKeywords.some((pattern) => pattern.test(text));
     }
 
     /**
      * Try to extract a FlowQuery statement that's not in a code block.
      */
     private extractInlineQuery(text: string): string | null {
-        const lines = text.split('\n');
+        const lines = text.split("\n");
         const queryLines: string[] = [];
         let inQuery = false;
 
         for (const line of lines) {
             const trimmedLine = line.trim();
-            
+
             if (!inQuery && this.startKeywords.test(trimmedLine)) {
                 inQuery = true;
                 queryLines.push(trimmedLine);
             } else if (inQuery) {
                 // Check if this line continues the query
-                if (this.continueKeywords.test(trimmedLine) || 
-                    trimmedLine.startsWith('{') ||
-                    trimmedLine.startsWith('}') ||
-                    trimmedLine === '' ||
+                if (
+                    this.continueKeywords.test(trimmedLine) ||
+                    trimmedLine.startsWith("{") ||
+                    trimmedLine.startsWith("}") ||
+                    trimmedLine === "" ||
                     /^[A-Za-z_][A-Za-z0-9_]*\./.test(trimmedLine) ||
-                    /^\s+/.test(line)) {  // Indented line
+                    /^\s+/.test(line)
+                ) {
+                    // Indented line
                     queryLines.push(trimmedLine);
                 } else {
                     // End of query
@@ -185,7 +190,7 @@ export class FlowQueryExtractor {
             }
         }
 
-        const query = queryLines.join('\n').trim();
+        const query = queryLines.join("\n").trim();
         return query && this.isLikelyFlowQuery(query) ? query : null;
     }
 }

package/misc/apps/RAG/src/utils/Llm.ts

@@ -0,0 +1,248 @@
+/**
+ * LLM (Large Language Model) Client
+ *
+ * Provides functions for making OpenAI-compatible API calls.
+ * Supports both regular and streaming responses.
+ */
+import { getStoredApiConfig } from "../components/ApiKeySettings";
+
+export interface LlmOptions {
+    /** System prompt to use */
+    systemPrompt?: string;
+    /** Conversation history */
+    messages?: Array<{ role: "user" | "assistant" | "system"; content: string }>;
+    /** Model to use (defaults to stored model or gpt-4o-mini) */
+    model?: string;
+    /** Temperature for response generation (0-2, default 0.7) */
+    temperature?: number;
+    /** Maximum tokens in response */
+    maxTokens?: number;
+    /** API key override (uses stored key if not provided) */
+    apiKey?: string;
+    /** Organization ID override (uses stored org if not provided) */
+    organizationId?: string;
+    /** Base URL for API (defaults to OpenAI) */
+    baseUrl?: string;
+}
+
+export interface LlmMessage {
+    role: "user" | "assistant" | "system";
+    content: string;
+}
+
+export interface LlmChoice {
+    index: number;
+    message: LlmMessage;
+    finish_reason: string;
+}
+
+export interface LlmResponse {
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+    choices: LlmChoice[];
+    usage?: {
+        prompt_tokens: number;
+        completion_tokens: number;
+        total_tokens: number;
+    };
+}
+
+export interface LlmStreamChoice {
+    index: number;
+    delta: {
+        role?: string;
+        content?: string;
+    };
+    finish_reason: string | null;
+}
+
+export interface LlmStreamChunk {
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+    choices: LlmStreamChoice[];
+}
+
+/**
+ * Get the API configuration, merging stored config with provided options.
+ */
+function getApiConfig(options: LlmOptions): {
+    apiKey: string;
+    orgId?: string;
+    model: string;
+    baseUrl: string;
+} {
+    const storedConfig = getStoredApiConfig();
+
+    const apiKey = options.apiKey || storedConfig?.apiKey;
+    if (!apiKey) {
+        throw new Error("No API key configured. Please set your OpenAI API key in settings.");
+    }
+
+    return {
+        apiKey,
+        orgId: options.organizationId || storedConfig?.organizationId,
+        model: options.model || storedConfig?.model || "gpt-4o-mini",
+        baseUrl: options.baseUrl || "https://api.openai.com/v1",
+    };
+}
+
+/**
+ * Build the messages array for the API request.
+ */
+function buildMessages(content: string, options: LlmOptions): LlmMessage[] {
+    const messages: LlmMessage[] = [];
+
+    // Add system prompt if provided
+    if (options.systemPrompt) {
+        messages.push({
+            role: "system",
+            content: options.systemPrompt,
+        });
+    }
+
+    // Add conversation history if provided
+    if (options.messages && options.messages.length > 0) {
+        messages.push(...options.messages);
+    }
+
+    // Add the current user message
+    messages.push({
+        role: "user",
+        content,
+    });
+
+    return messages;
+}
+
+/**
+ * Make a non-streaming LLM API call.
+ *
+ * @param content - The user's message content
+ * @param options - LLM options including system prompt and conversation history
+ * @returns Promise resolving to the LLM response
+ */
+export async function llm(content: string, options: LlmOptions = {}): Promise<LlmResponse> {
+    const config = getApiConfig(options);
+    const messages = buildMessages(content, options);
+
+    const headers: Record<string, string> = {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${config.apiKey}`,
+    };
+
+    if (config.orgId) {
+        headers["OpenAI-Organization"] = config.orgId;
+    }
+
+    const response = await fetch(`${config.baseUrl}/chat/completions`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+            model: config.model,
+            messages,
+            temperature: options.temperature ?? 0.7,
+            max_tokens: options.maxTokens,
+        }),
+    });
+
+    if (!response.ok) {
+        const errorBody = await response.text();
+        throw new Error(`LLM API error (${response.status}): ${errorBody}`);
+    }
+
+    return response.json();
+}
+
+/**
+ * Make a streaming LLM API call.
+ *
+ * @param content - The user's message content
+ * @param options - LLM options including system prompt and conversation history
+ * @yields LLM stream chunks as they arrive
+ */
+export async function* llmStream(
+    content: string,
+    options: LlmOptions = {}
+): AsyncGenerator<LlmStreamChunk, void, unknown> {
+    const config = getApiConfig(options);
+    const messages = buildMessages(content, options);
+
+    const headers: Record<string, string> = {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${config.apiKey}`,
+    };
+
+    if (config.orgId) {
+        headers["OpenAI-Organization"] = config.orgId;
+    }
+
+    const response = await fetch(`${config.baseUrl}/chat/completions`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+            model: config.model,
+            messages,
+            temperature: options.temperature ?? 0.7,
+            max_tokens: options.maxTokens,
+            stream: true,
+        }),
+    });
+
+    if (!response.ok) {
+        const errorBody = await response.text();
+        throw new Error(`LLM API error (${response.status}): ${errorBody}`);
+    }
+
+    if (!response.body) {
+        throw new Error("Response body is null");
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+
+            if (done) {
+                break;
+            }
+
+            buffer += decoder.decode(value, { stream: true });
+
+            // Process complete SSE messages
+            const lines = buffer.split("\n");
+            buffer = lines.pop() || ""; // Keep incomplete line in buffer
+
+            for (const line of lines) {
+                const trimmedLine = line.trim();
+
+                if (!trimmedLine || trimmedLine === ":") {
+                    continue;
+                }
+
+                if (trimmedLine === "data: [DONE]") {
+                    return;
+                }
+
+                if (trimmedLine.startsWith("data: ")) {
+                    const jsonStr = trimmedLine.slice(6);
+                    try {
+                        const chunk = JSON.parse(jsonStr) as LlmStreamChunk;
+                        yield chunk;
+                    } catch {
+                        // Skip invalid JSON chunks
+                        console.warn("Failed to parse LLM stream chunk:", jsonStr);
+                    }
+                }
+            }
+        }
+    } finally {
+        reader.releaseLock();
+    }
+}

package/misc/apps/RAG/src/utils/index.ts

@@ -2,8 +2,11 @@
  * Utils module exports
  */
 
-export { FlowQueryExecutor } from './FlowQueryExecutor';
-export type { FlowQueryExecutionResult } from './FlowQueryExecutor';
+export { FlowQueryExecutor } from "./FlowQueryExecutor";
+export type { FlowQueryExecutionResult } from "./FlowQueryExecutor";
 
-export { FlowQueryExtractor, extractFlowQuery, extractAllFlowQueries } from './FlowQueryExtractor';
-export type { FlowQueryExtraction } from './FlowQueryExtractor';
+export { FlowQueryExtractor, extractFlowQuery, extractAllFlowQueries } from "./FlowQueryExtractor";
+export type { FlowQueryExtraction } from "./FlowQueryExtractor";
+
+export { llm, llmStream } from "./Llm";
+export type { LlmOptions, LlmResponse, LlmStreamChunk } from "./Llm";

package/misc/apps/RAG/tsconfig.json

@@ -1,10 +1,11 @@
 {
   "compilerOptions": {
     "outDir": "./dist",
-    "rootDir": "./src",
+    "rootDir": ".",
     "target": "ES2020",
     "module": "ESNext",
     "moduleResolution": "bundler",
+    "experimentalDecorators": true,
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
     "strict": true,
@@ -14,8 +15,8 @@
     "resolveJsonModule": true,
     "jsx": "react-jsx",
     "lib": ["DOM", "DOM.Iterable", "ESNext"],
-    "types": ["node"]
+    "types": ["node", "jest"]
   },
-  "include": ["src/**/*"],
+  "include": ["src/**/*", "data/**/*"],
   "exclude": ["node_modules", "dist"]
 }

package/misc/apps/RAG/webpack.config.js

@@ -1,43 +1,43 @@
-const path = require('path');
-const HtmlWebpackPlugin = require('html-webpack-plugin');
+const path = require("path");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
 
 module.exports = {
-  mode: 'development',
-  entry: './src/index.tsx',
-  target: 'web',
-  output: {
-    filename: 'bundle.js',
-    path: path.resolve(__dirname, 'dist'),
-    clean: true
-  },
-  resolve: {
-    extensions: ['.tsx', '.ts', '.js', '.jsx']
-  },
-  module: {
-    rules: [
-      {
-        test: /\.tsx?$/,
-        use: 'ts-loader',
-        exclude: /node_modules/
-      },
-      {
-        test: /\.css$/,
-        use: ['style-loader', 'css-loader']
-      }
-    ]
-  },
-  plugins: [
-    new HtmlWebpackPlugin({
-      template: './public/index.html'
-    }),
-  ],
-  devServer: {
-    static: './dist',
-    port: 3000,
-    open: true,
-    hot: true
-  },
-  optimization: {
-    minimize: false
-  }
+    mode: "development",
+    entry: "./src/index.tsx",
+    target: "web",
+    output: {
+        filename: "bundle.js",
+        path: path.resolve(__dirname, "dist"),
+        clean: true,
+    },
+    resolve: {
+        extensions: [".tsx", ".ts", ".js", ".jsx"],
+    },
+    module: {
+        rules: [
+            {
+                test: /\.tsx?$/,
+                use: "ts-loader",
+                exclude: /node_modules/,
+            },
+            {
+                test: /\.css$/,
+                use: ["style-loader", "css-loader"],
+            },
+        ],
+    },
+    plugins: [
+        new HtmlWebpackPlugin({
+            template: "./public/index.html",
+        }),
+    ],
+    devServer: {
+        static: ["./dist", { directory: path.resolve(__dirname, "data"), publicPath: "/data" }],
+        port: 3000,
+        open: true,
+        hot: true,
+    },
+    optimization: {
+        minimize: false,
+    },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flowquery",
-  "version": "1.0.25",
+  "version": "1.0.26",
   "description": "A declarative query language for data processing pipelines.",
   "main": "dist/index.node.js",
   "types": "dist/index.node.d.ts",