@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.1

package/src/dev/README.md

@@ -1,126 +0,0 @@
-# Development Tools
-
-This directory contains development tools for the RAG Pipeline Utils library.
-
-## Available Tools
-
-### Hot Reload Manager (`hot-reload.js`)
-
-Provides automatic reloading of plugins and configuration files during development.
-
-**Features:**
-
-- File watching with chokidar
-- Plugin hot-reloading with module cache clearing
-- Configuration hot-reloading with validation
-- State preservation and restoration
-- Lifecycle hooks (beforeReload, afterReload, onError, etc.)
-- Statistics tracking and history
-- Error handling with notifications
-
-**Usage:**
-
-```javascript
-const { createHotReloadManager } = require("./hot-reload");
-
-const hotReload = createHotReloadManager({
-  watchPaths: ["./src/plugins"],
-  configPath: ".ragrc.json",
-  verbose: true,
-});
-
-await hotReload.start();
-```
-
-### Development Server (`dev-server.js`)
-
-Integrated development server with hot reload support.
-
-**Features:**
-
-- Hot reload integration
-- Real-time notifications
-- Statistics dashboard
-- Event-driven architecture
-- Graceful shutdown
-- Pipeline integration
-
-**Usage:**
-
-```javascript
-const { createDevServer } = require("./dev-server");
-
-const devServer = createDevServer({
-  port: 3000,
-  watchPaths: ["./src/plugins"],
-  autoReload: true,
-});
-
-await devServer.start();
-```
-
-## Examples
-
-See `examples/hot-reload-usage.js` for comprehensive examples including:
-
-1. Basic hot reload setup
-2. Integration with plugin registry
-3. Development server usage
-4. Pipeline integration
-5. Custom state preservation
-6. Manual reload triggering
-7. Configuration validation
-8. Complete development workflows
-
-## Documentation
-
-Full documentation is available at `docs/hot-reload.md`.
-
-## Testing
-
-Tests are located in `__tests__/dev/hot-reload.test.js`.
-
-Run tests:
-
-```bash
-npm test -- __tests__/dev/hot-reload.test.js
-```
-
-## API Reference
-
-### Hot Reload Manager
-
-- `start()` - Start watching for changes
-- `stop()` - Stop watching
-- `triggerReload(filePath, type)` - Manually trigger reload
-- `registerHook(hookName, callback)` - Register lifecycle hook
-- `getStats()` - Get reload statistics
-- `getHistory()` - Get reload history
-
-### Development Server
-
-- `start()` - Start dev server
-- `stop()` - Stop dev server
-- `getStatus()` - Get server status
-- `getStatistics()` - Get formatted statistics
-- `triggerReload(filePath, type)` - Manually trigger reload
-- `registerHook(hookName, callback)` - Register lifecycle hook
-
-## Events
-
-Both tools emit various events:
-
-- `started` / `stopped`
-- `plugin:reloaded` / `plugin:reload-failed`
-- `config:reloaded` / `config:reload-failed`
-- `plugin:added` / `plugin:removed`
-- `error`
-- `notification` (DevServer only)
-
-## Dependencies
-
-- `chokidar` - File system watching
-
-## License
-
-GPL-3.0

package/src/dev/examples/hot-reload-usage.js

@@ -1,355 +0,0 @@
-'use strict';
-
-/**
- * Hot Reload Usage Examples
- *
- * This file demonstrates various ways to use the hot reload functionality
- * in development environments.
- */
-
-const { createHotReloadManager } = require('../hot-reload');
-const { createDevServer } = require('../dev-server');
-const { createRagPipeline } = require('../../core/create-pipeline');
-const pluginRegistry = require('../../core/plugin-registry');
-
-// ============================================================================
-// Example 1: Basic Hot Reload Setup
-// ============================================================================
-
-function basicHotReloadExample() {
-  console.log('\n=== Example 1: Basic Hot Reload ===\n');
-
-  // Create hot reload manager
-  const hotReload = createHotReloadManager({
-    enabled: true,
-    watchPaths: ['./src/plugins', './src/mocks'],
-    configPath: '.ragrc.json',
-    verbose: true,
-  });
-
-  // Listen to events
-  hotReload.on('plugin:reloaded', ({ path, duration }) => {
-    console.log(`Plugin reloaded: ${path} in ${duration}ms`);
-  });
-
-  hotReload.on('config:reloaded', ({ config }) => {
-    console.log('Configuration reloaded:', config.metadata?.name);
-  });
-
-  // Start watching
-  hotReload.start().then(() => {
-    console.log('Hot reload started');
-  });
-
-  // To stop:
-  // hotReload.stop();
-}
-
-// ============================================================================
-// Example 2: Hot Reload with Plugin Registry Integration
-// ============================================================================
-
-async function hotReloadWithRegistryExample() {
-  console.log('\n=== Example 2: Hot Reload with Plugin Registry ===\n');
-
-  // Create hot reload manager with plugin registry
-  const hotReload = createHotReloadManager({
-    enabled: true,
-    watchPaths: ['./src/plugins'],
-    pluginRegistry: pluginRegistry,
-    verbose: true,
-    preserveState: true, // Preserve plugin state during reloads
-  });
-
-  // Register hooks for advanced control
-  hotReload.registerHook('beforeReload', async (data) => {
-    console.log(`About to reload ${data.type}: ${data.path}`);
-    // Perform any cleanup or preparation
-  });
-
-  hotReload.registerHook('afterReload', async (data) => {
-    console.log(`Reload completed for ${data.type}: ${data.path}`);
-    // Re-register with plugin registry, update references, etc.
-  });
-
-  hotReload.registerHook('onError', async ({ message, error }) => {
-    console.error(`Reload error: ${message}`, error);
-    // Send notification, log to service, etc.
-  });
-
-  await hotReload.start();
-
-  // Get statistics
-  setInterval(() => {
-    const stats = hotReload.getStats();
-    console.log('Reload Stats:', {
-      total: stats.totalReloads,
-      successful: stats.successfulReloads,
-      failed: stats.failedReloads,
-      avgTime: Math.round(stats.averageReloadTime) + 'ms',
-    });
-  }, 60000); // Every minute
-}
-
-// ============================================================================
-// Example 3: Development Server with Hot Reload
-// ============================================================================
-
-async function devServerExample() {
-  console.log('\n=== Example 3: Development Server ===\n');
-
-  // Create development server with integrated hot reload
-  const devServer = createDevServer({
-    port: 3000,
-    host: 'localhost',
-    watchPaths: ['./src/plugins', './src/mocks'],
-    configPath: '.ragrc.json',
-    verbose: true,
-    autoReload: true,
-    notifyOnReload: true,
-  });
-
-  // Listen to server events
-  devServer.on('started', () => {
-    console.log('Development server started');
-    console.log(devServer.getStatistics());
-  });
-
-  devServer.on('plugin:reloaded', ({ path, duration }) => {
-    console.log(`✓ Plugin reloaded: ${path} (${duration}ms)`);
-  });
-
-  devServer.on('config:reloaded', ({ path, duration }) => {
-    console.log(`✓ Config reloaded: ${path} (${duration}ms)`);
-  });
-
-  devServer.on('notification', ({ message, type }) => {
-    console.log(`[${type.toUpperCase()}] ${message}`);
-  });
-
-  // Start the server
-  await devServer.start();
-
-  // Display statistics periodically
-  setInterval(() => {
-    console.log(devServer.getStatistics());
-  }, 300000); // Every 5 minutes
-
-  // Handle graceful shutdown
-  process.on('SIGINT', async () => {
-    console.log('\nShutting down...');
-    await devServer.stop();
-    process.exit(0);
-  });
-}
-
-// ============================================================================
-// Example 4: Hot Reload with RAG Pipeline
-// ============================================================================
-
-async function hotReloadWithPipelineExample() {
-  console.log('\n=== Example 4: Hot Reload with Pipeline ===\n');
-
-  // Create pipeline
-  const pipeline = await createRagPipeline({
-    configPath: '.ragrc.json',
-  });
-
-  // Create dev server with pipeline integration
-  const devServer = createDevServer({
-    watchPaths: ['./src/plugins', './src/mocks'],
-    configPath: '.ragrc.json',
-    verbose: true,
-    pluginRegistry: pluginRegistry,
-    pipeline: pipeline,
-  });
-
-  // When config changes, pipeline will be reinitialized automatically
-  devServer.on('config:reloaded', async () => {
-    console.log('Pipeline configuration updated');
-    // Pipeline is automatically reinitialized by dev server
-  });
-
-  // When plugins change, they're hot-reloaded
-  devServer.on('plugin:reloaded', async ({ path }) => {
-    console.log(`Plugin ${path} updated and reloaded`);
-    // Plugin is automatically updated in registry
-  });
-
-  await devServer.start();
-}
-
-// ============================================================================
-// Example 5: Custom State Preservation
-// ============================================================================
-
-async function customStatePreservationExample() {
-  console.log('\n=== Example 5: Custom State Preservation ===\n');
-
-  const hotReload = createHotReloadManager({
-    enabled: true,
-    watchPaths: ['./src/plugins'],
-    preserveState: true,
-    verbose: true,
-  });
-
-  // Hook into state preservation
-  hotReload.registerHook('onStatePreserve', async ({ path, state }) => {
-    console.log(`Preserving state for ${path}:`, state);
-    // You can save state to external storage here
-  });
-
-  hotReload.registerHook('onStateRestore', async ({ plugin, state }) => {
-    console.log('Restoring state:', state);
-    // You can restore state from external storage here
-  });
-
-  await hotReload.start();
-}
-
-// ============================================================================
-// Example 6: Manual Reload Triggering
-// ============================================================================
-
-async function manualReloadExample() {
-  console.log('\n=== Example 6: Manual Reload ===\n');
-
-  const devServer = createDevServer({
-    watchPaths: ['./src/plugins'],
-    autoReload: false, // Disable automatic reloading
-    verbose: true,
-  });
-
-  await devServer.start();
-
-  // Manually trigger reloads
-  await devServer.triggerReload('./src/plugins/my-plugin.js', 'plugin');
-  await devServer.triggerReload('.ragrc.json', 'config');
-
-  console.log('Manual reloads completed');
-}
-
-// ============================================================================
-// Example 7: Configuration Validation
-// ============================================================================
-
-async function configValidationExample() {
-  console.log('\n=== Example 7: Config Validation ===\n');
-
-  const hotReload = createHotReloadManager({
-    enabled: true,
-    configPath: '.ragrc.json',
-    verbose: true,
-    // Custom config validator
-    configValidator: (config) => {
-      const errors = [];
-
-      // Validate required fields
-      if (!config.metadata || !config.metadata.name) {
-        errors.push({ message: 'metadata.name is required' });
-      }
-
-      if (!config.pipeline || !config.pipeline.stages) {
-        errors.push({ message: 'pipeline.stages is required' });
-      }
-
-      return {
-        valid: errors.length === 0,
-        errors,
-      };
-    },
-  });
-
-  hotReload.on('config:reload-failed', ({ error }) => {
-    console.error('Configuration validation failed:', error);
-  });
-
-  await hotReload.start();
-}
-
-// ============================================================================
-// Example 8: Development Workflow Integration
-// ============================================================================
-
-async function developmentWorkflowExample() {
-  console.log('\n=== Example 8: Complete Development Workflow ===\n');
-
-  // Create a complete development environment
-  const devServer = createDevServer({
-    port: 3000,
-    host: 'localhost',
-    watchPaths: ['./src/plugins', './src/mocks', './src/config'],
-    configPath: '.ragrc.json',
-    verbose: true,
-    autoReload: true,
-    notifyOnReload: true,
-    pluginRegistry: pluginRegistry,
-  });
-
-  // Add custom hooks
-  devServer.registerHook('beforeReload', async (data) => {
-    console.log(`[BEFORE] Reloading ${data.type}: ${data.path}`);
-    // Run pre-reload tasks (tests, linting, etc.)
-  });
-
-  devServer.registerHook('afterReload', async (data) => {
-    console.log(`[AFTER] Reloaded ${data.type}: ${data.path}`);
-    // Run post-reload tasks (smoke tests, notifications, etc.)
-  });
-
-  // Start server
-  await devServer.start();
-
-  // Log status every 5 minutes
-  setInterval(() => {
-    const status = devServer.getStatus();
-    console.log('\n=== Server Status ===');
-    console.log(`Uptime: ${Math.floor(status.uptime / 1000)}s`);
-    console.log(`Total Reloads: ${status.hotReload.stats.totalReloads}`);
-    console.log(
-      `Success Rate: ${
-        status.hotReload.stats.totalReloads > 0
-          ? Math.round(
-              (status.hotReload.stats.successfulReloads /
-                status.hotReload.stats.totalReloads) *
-                100,
-            ) + '%'
-          : 'N/A'
-      }`,
-    );
-  }, 300000);
-
-  // Handle shutdown
-  process.on('SIGINT', async () => {
-    console.log('\n\nShutting down development server...');
-    console.log(devServer.getStatistics());
-    await devServer.stop();
-    process.exit(0);
-  });
-}
-
-// ============================================================================
-// Run Examples
-// ============================================================================
-
-// Uncomment the example you want to run:
-
-// basicHotReloadExample();
-// hotReloadWithRegistryExample();
-// devServerExample();
-// hotReloadWithPipelineExample();
-// customStatePreservationExample();
-// manualReloadExample();
-// configValidationExample();
-// developmentWorkflowExample();
-
-module.exports = {
-  basicHotReloadExample,
-  hotReloadWithRegistryExample,
-  devServerExample,
-  hotReloadWithPipelineExample,
-  customStatePreservationExample,
-  manualReloadExample,
-  configValidationExample,
-  developmentWorkflowExample,
-};

package/src/mocks/openai-embedder.js

@@ -1,119 +0,0 @@
-/**
- * Version: 2.0.0
- * Description: Mock implementation of OpenAI embedder for testing and development
- * Author: Ali Kahwaji
- */
-
-class OpenAIEmbedder {
-  constructor(_options = {}) {
-    this.apiKey = _options.apiKey || "mock-api-key";
-    this.model = _options.model || "text-embedding-ada-002";
-    this.dimensions = _options.dimensions || 1536;
-    this.options = _options;
-
-    // Configurable batch size from environment or options
-    this.batchSize =
-      parseInt(process.env.RAG_EMBEDDER_BATCH_SIZE) ||
-      _options.batchSize ||
-      100; // Default batch size for OpenAI API
-  }
-
-  /**
-   * Generate embeddings for an array of text chunks
-   * @param {string[]} chunks - Array of text chunks to embed
-   * @returns {Promise<number[][]>} Array of embedding vectors
-   */
-  async embed(chunks) {
-    if (!Array.isArray(chunks)) {
-      throw new Error("embed() expects an array of strings");
-    }
-
-    // Process in batches for better memory management and API rate limiting
-    const results = [];
-    for (let i = 0; i < chunks.length; i += this.batchSize) {
-      const batch = chunks.slice(i, i + this.batchSize);
-      const batchResults = await this._processBatch(batch);
-      results.push(...batchResults);
-
-      // Optional progress callback
-      if (this.options.onProgress) {
-        this.options.onProgress({
-          processed: Math.min(i + this.batchSize, chunks.length),
-          total: chunks.length,
-          stage: "embedding",
-        });
-      }
-
-      // Simulate API rate limiting delay
-      if (chunks.length > this.batchSize) {
-        await new Promise((resolve) => setTimeout(resolve, 50));
-      }
-    }
-
-    return results;
-  }
-
-  /**
-   * Process a batch of chunks
-   * @private
-   * @param {string[]} batch - Batch of text chunks
-   * @returns {Promise<number[][]>} Array of embedding vectors
-   */
-  async _processBatch(batch) {
-    // Return mock embeddings - deterministic based on content for testing
-    return batch.map((chunk) => this.#generateMockEmbedding(chunk));
-  }
-
-  /**
-   * Generate embedding for a single query string
-   * @param {string} query - Query string to embed
-   * @returns {Promise<number[]>} Embedding vector
-   */
-  async embedQuery(query) {
-    if (typeof query !== "string") {
-      throw new Error("embedQuery() expects a string");
-    }
-
-    return this.#generateMockEmbedding(query);
-  }
-
-  /**
-   * Generate a deterministic mock embedding based on input text
-   * @private
-   * @param {string} text - Input text
-   * @returns {number[]} Mock embedding vector
-   */
-  #generateMockEmbedding(text) {
-    // Create deterministic "embedding" based on text content
-    const hash = this.#simpleHash(text);
-    const embedding = [];
-
-    for (let i = 0; i < this.dimensions; i++) {
-      // Generate pseudo-random values based on hash and index
-      const value = Math.sin(hash + i) * 0.5;
-      embedding.push(value);
-    }
-
-    return embedding;
-  }
-
-  /**
-   * Simple hash function for deterministic mock data
-   * @private
-   * @param {string} str - Input string
-   * @returns {number} Hash value
-   */
-  #simpleHash(str) {
-    let hash = 0;
-    for (let i = 0; i < str.length; i++) {
-      const char = str.charCodeAt(i);
-      hash = (hash << 5) - hash + char;
-      hash = hash & hash; // Convert to 32-bit integer
-    }
-    return hash;
-  }
-}
-
-module.exports = {
-  OpenAIEmbedder,
-};

package/src/mocks/openai-llm.js

@@ -1,77 +0,0 @@
-/**
- * Version: 2.0.0
- * File: /src/mocks/openai-llm.js
- * Description: Mock implementation of an OpenAI LLM with streaming support
- * Author: Ali Kahwaji
- */
-
-class OpenAILLM {
-  constructor(_options = {}) {
-    this.streamDelay = _options.streamDelay || 50; // ms between tokens
-    this.model = _options.model || "gpt-3.5-turbo";
-  }
-
-  /**
-   * Generates a complete response from input prompt and context
-   * @param {string} prompt
-   * @param {Array<{ text: string }>} context
-   * @returns {Promise<string>}
-   */
-  async generate(prompt, context) {
-    const contextText = Array.isArray(context)
-      ? context.map((d) => d.text || d).join(", ")
-      : "";
-    return `Generated answer using context: ${contextText}. Query: "${prompt}". This is a comprehensive response that addresses the user's question based on the retrieved information.`;
-  }
-
-  /**
-   * Generates a streaming response token-by-token
-   * @param {string} prompt
-   * @param {Array<{ text: string }>} context
-   * @returns {AsyncIterable<string>} Stream of tokens
-   */
-  async *generateStream(prompt, context) {
-    const fullResponse = await this.generate(prompt, context);
-    const tokens = this.#tokenize(fullResponse);
-
-    for (const token of tokens) {
-      // Simulate network delay between tokens
-      await this.#delay(this.streamDelay);
-      yield token;
-    }
-  }
-
-  /**
-   * Simple tokenization for mock streaming
-   * @private
-   * @param {string} text - Text to tokenize
-   * @returns {string[]} Array of tokens
-   */
-  #tokenize(text) {
-    // Simple word-based tokenization for demo purposes
-    // In real implementation, this would use proper tokenization
-    return text.split(/\s+/).map((word) => word + " ");
-  }
-
-  /**
-   * Utility delay function for simulating streaming
-   * @private
-   * @param {number} ms - Milliseconds to delay
-   * @returns {Promise<void>}
-   */
-  #delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
-  /**
-   * Check if this LLM _instance supports streaming
-   * @returns {boolean}
-   */
-  supportsStreaming() {
-    return true;
-  }
-}
-
-// Default export
-
-module.exports = OpenAILLM;

package/src/mocks/pdf-loader.js

@@ -1,21 +0,0 @@
-/**
- * Version: 0.1.0
- * File: /src/mocks/pdf-loader.js
- * Description: Mock implementation of a PDF document loader
- * Author: Ali Kahwaji
- */
-
-class PDFLoader {
-  /**
-   * Loads a mock PDF file and returns an array of chunked content.
-   * @param {string} _filePath - Path to the PDF file
-   * @returns {Promise<Array<{ chunk: () => string[] }>>}
-   */
-  async load(_filePath) {
-    return [{ chunk: () => [`Test loaded: ${_filePath}`] }];
-  }
-}
-
-// Default export
-
-module.exports = PDFLoader;