@devilsdev/rag-pipeline-utils 2.2.1 → 2.3.1

package/src/dag/dag-engine.js

@@ -6,10 +6,23 @@
  */
 
 const DAGNode = require('./dag-node.js');
+const {
+  buildAdjacency,
+  getSinkIds,
+  topologicalSort,
+  validateDAG,
+  validateTopology,
+} = require('./core/topology-validation.js');
+const ExecutionScheduler = require('./core/execution-scheduler.js');
+const ErrorContext = require('./core/error-context.js');
 
 class DAG {
   constructor() {
     this.nodes = new Map();
+    this.scheduler = new ExecutionScheduler({
+      getSinkIds: (fwd) => getSinkIds(fwd),
+    });
+    this.errorContext = new ErrorContext();
   }
 
   addNode(id, _fn, options = {}) {
@@ -87,89 +100,7 @@ class DAG {
    * @returns {Promise<void>}
    */
   async _runAllNodes(order, context) {
-    const {
-      concurrency,
-      results,
-      errors,
-      fwd,
-      continueOnError,
-      requiredIds,
-      seed,
-      gracefulDegradation,
-      requiredNodes,
-      retryFailedNodes,
-      maxRetries,
-    } = context;
-
-    const semaphore = concurrency ? new Array(concurrency).fill(null) : null;
-    let semaphoreIndex = 0;
-
-    const executeWithSemaphore = async (node) => {
-      if (semaphore) {
-        // Wait for available slot
-        while (semaphore[semaphoreIndex % semaphore.length] !== null) {
-          await new Promise((resolve) => setTimeout(resolve, 10));
-        }
-        const slotIndex = semaphoreIndex % semaphore.length;
-        semaphoreIndex++;
-
-        try {
-          const result = await this.executeNode(node, {
-            results,
-            errors,
-            fwd,
-            continueOnError,
-            requiredIds,
-            seed,
-            gracefulDegradation,
-            requiredNodes,
-            retryFailedNodes,
-            maxRetries,
-          });
-          // Only set result if it's not undefined (failed optional nodes return undefined)
-          if (result !== undefined) {
-            results.set(node.id, result);
-          }
-        } finally {
-          semaphore[slotIndex] = null;
-        }
-      } else {
-        const result = await this.executeNode(node, {
-          results,
-          errors,
-          fwd,
-          continueOnError,
-          requiredIds,
-          seed,
-          gracefulDegradation,
-          requiredNodes,
-          retryFailedNodes,
-          maxRetries,
-        });
-        // Only set result if it's not the special failed-optional-node symbol
-        if (
-          typeof result !== 'symbol' ||
-          result.description !== 'failed-optional-node'
-        ) {
-          results.set(node.id, result);
-        }
-      }
-    };
-
-    // Execute nodes respecting dependencies
-    for (const node of order) {
-      // Check if all dependencies are satisfied
-      const canExecute = this._canExecuteNode(node, {
-        results,
-        errors,
-        gracefulDegradation,
-        requiredNodes,
-      });
-
-      if (canExecute) {
-        await executeWithSemaphore(node);
-      }
-    }
+    return this.scheduler.scheduleExecution(order, context);
   }
 
   /**
@@ -179,23 +110,7 @@ class DAG {
    * @returns {boolean}
    */
   _canExecuteNode(node, context) {
-    const { results, errors, gracefulDegradation, requiredNodes } = context;
-
-    return node.inputs.every((dep) => {
-      // If dependency has result or error, it's satisfied
-      if (results.has(dep.id) || errors.has(dep.id)) {
-        return true;
-      }
-      // If graceful degradation is enabled and dependency is not required, it's satisfied
-      if (
-        gracefulDegradation &&
-        requiredNodes &&
-        !requiredNodes.includes(dep.id)
-      ) {
-        return true;
-      }
-      return false;
-    });
+    return this.scheduler.canExecuteNode(node, context);
   }
 
   /**
@@ -221,41 +136,7 @@ class DAG {
    * @throws {Error} - Appropriately wrapped error
    */
   _handleExecutionError(error) {
-    // Error policy: wrap everything in try/catch
-    // If caught error is a node error (message starts with "Node " OR error.nodeId) → rethrow unchanged
-    if (error.message.startsWith('Node ') || error.nodeId) {
-      throw error;
-    }
-
-    // If caught error is validation or aggregate → wrap once
-    if (
-      error.cycle ||
-      /^DAG validation failed/.test(error.message) ||
-      error.errors
-    ) {
-      const wrappedError = new Error(`DAG execution failed: ${error.message}`);
-      if (error.cycle) wrappedError.cycle = error.cycle;
-      if (error.errors) wrappedError.errors = error.errors;
-      if (error.nodeId) wrappedError.nodeId = error.nodeId;
-      if (error.timestamp) wrappedError.timestamp = error.timestamp;
-      throw wrappedError;
-    }
-
-    // Do NOT wrap timeout errors or 'no sink nodes' errors
-    if (
-      error.message === 'Execution timeout' ||
-      error.message === 'DAG has no sink nodes - no final output available'
-    ) {
-      throw error;
-    }
-
-    // Otherwise wrap with 'DAG execution failed: ...'
-    const wrappedError = new Error(`DAG execution failed: ${error.message}`);
-    if (error.errors) wrappedError.errors = error.errors;
-    if (error.nodeId) wrappedError.nodeId = error.nodeId;
-    if (error.timestamp) wrappedError.timestamp = error.timestamp;
-    if (error.cycle) wrappedError.cycle = error.cycle;
-    throw wrappedError;
+    throw this.errorContext.wrapExecutionError(error);
   }
 
   /**
@@ -265,16 +146,7 @@ class DAG {
    * @returns {Promise<void>}
    */
   async _executeWithTimeout(runFn, timeout) {
-    if (timeout && timeout > 0) {
-      await Promise.race([
-        runFn(),
-        new Promise((_, reject) => {
-          setTimeout(() => reject(new Error('Execution timeout')), timeout);
-        }),
-      ]);
-    } else {
-      await runFn();
-    }
+    return this.scheduler.executeWithTimeout(runFn, timeout);
   }
 
   /**
@@ -294,10 +166,9 @@ class DAG {
     } = options;
 
     // Check for multiple errors first
-    if (errors.size >= 2) {
-      const agg = new Error('Multiple execution errors');
-      agg.errors = [...errors.values()].map((x) => x.cause || x);
-      throw agg;
+    const aggregatedError = this.errorContext.aggregateErrors(errors);
+    if (aggregatedError && errors.size >= 2) {
+      throw aggregatedError;
     }
 
     // Determine sink nodes and successful sinks
@@ -435,27 +306,7 @@ class DAG {
    * @returns {Object} - Object with fwd and rev adjacency maps
    */
   _buildAdjacency() {
-    const fwd = new Map(); // node -> Set of children
-    const rev = new Map(); // node -> Set of parents
-
-    // Initialize adjacency lists for all nodes
-    for (const node of this.nodes.values()) {
-      fwd.set(node.id, new Set());
-      rev.set(node.id, new Set());
-    }
-
-    // Populate adjacency lists based on node connections
-    for (const node of this.nodes.values()) {
-      for (const output of node.outputs) {
-        fwd.get(node.id).add(output.id);
-        // Only add reverse edge if the output node exists in the DAG
-        if (rev.has(output.id)) {
-          rev.get(output.id).add(node.id);
-        }
-      }
-    }
-
-    return { fwd, rev };
+    return buildAdjacency(this.nodes);
   }
 
   /**
@@ -464,13 +315,7 @@ class DAG {
    * @returns {Array} - Array of sink node IDs
    */
   _getSinkIds(fwd) {
-    const sinkIds = [];
-    for (const [nodeId, children] of fwd) {
-      if (children.size === 0) {
-        sinkIds.push(nodeId);
-      }
-    }
-    return sinkIds;
+    return getSinkIds(fwd);
   }
 
   /**
@@ -523,46 +368,7 @@ class DAG {
    * @throws {Error} - If validation fails
    */
   validate() {
-    if (this.nodes.size === 0) {
-      throw new Error('DAG is empty - no nodes to execute');
-    }
-
-    // Check for cycles using topological sort
-    try {
-      this.topoSort();
-    } catch (error) {
-      if (error.message.includes('Cycle detected')) {
-        // Extract cycle information from error message or use error.cycle if available
-        let nodes = error.cycle;
-        if (!nodes && error.message.includes('involving node:')) {
-          // Parse cycle from message if cycle array not available
-          const match = error.message.match(/involving node: (.+)/);
-          if (match) {
-            nodes = match[1].split(' -> ');
-          }
-        }
-        if (!nodes) {
-          nodes = ['unknown'];
-        }
-
-        const pretty = nodes.join(' -> ');
-        const err = new Error(
-          `DAG validation failed: DAG topological sort failed: Cycle detected involving node: ${pretty}`,
-        );
-        err.cycle = nodes;
-        throw err;
-      }
-      throw new Error(`DAG validation failed: ${error.message}`);
-    }
-
-    // Check for orphaned nodes (nodes with no path to any sink)
-    const { fwd } = this._buildAdjacency();
-    const sinkIds = this._getSinkIds(fwd);
-    if (sinkIds.length === 0) {
-      throw new Error('DAG has no sink nodes - no final output available');
-    }
-
-    return true;
+    return validateDAG(this.nodes);
   }
 
   /**
@@ -572,104 +378,7 @@ class DAG {
    * @returns {Promise<any>} Node execution result
    */
   async executeNode(node, context) {
-    const {
-      results,
-      errors,
-      fwd,
-      continueOnError,
-      requiredIds,
-      seed,
-      gracefulDegradation,
-      requiredNodes,
-    } = context;
-
-    // Handle undefined node.run function
-    if (typeof node.run !== 'function') {
-      throw new Error(`Node ${node.id} has no run function`);
-    }
-
-    // Determine criticality - default is critical unless explicitly marked optional
-    // OR if gracefulDegradation is enabled and node is not in requiredNodes
-    // OR if there are multiple sink nodes (individual sink failures should be non-critical)
-    const sinkIds = this._getSinkIds(fwd);
-    const isMultipleSinks = sinkIds.length > 1;
-    const isSinkNode = sinkIds.includes(node.id);
-
-    const isNonCritical =
-      node?.optional === true ||
-      node?.isOptional === true ||
-      node?.critical === false ||
-      (gracefulDegradation &&
-        requiredNodes &&
-        !requiredNodes.includes(node.id)) ||
-      (isMultipleSinks && isSinkNode); // Sink nodes are non-critical when there are multiple sinks
-
-    // Prepare input based on dependencies
-    let input;
-    if (node.inputs.length === 0) {
-      input = seed;
-    } else if (node.inputs.length === 1) {
-      input = results.get(node.inputs[0].id);
-    } else {
-      input = node.inputs.map((inputNode) => {
-        // For graceful degradation, return undefined for failed optional dependencies
-        if (
-          !results.has(inputNode.id) &&
-          gracefulDegradation &&
-          requiredNodes &&
-          !requiredNodes.includes(inputNode.id)
-        ) {
-          return undefined;
-        }
-        return results.get(inputNode.id);
-      });
-    }
-
-    // Retry logic - use global options if available, otherwise node-specific
-    const { retryFailedNodes, maxRetries: globalMaxRetries } = context;
-    const nodeMaxRetries = retryFailedNodes
-      ? Math.max(0, globalMaxRetries ?? 3)
-      : Math.max(0, node?.retry?.retries ?? 0);
-    const delayMs = Math.max(0, node?.retry?.delayMs ?? 0);
-    let attempt = 0;
-    let keepRetrying = true;
-
-    while (keepRetrying) {
-      try {
-        // Execute the node
-        const output = await node.run(input);
-        if (errors.has(node.id)) errors.delete(node.id);
-        return output;
-      } catch (e) {
-        attempt += 1;
-        if (attempt <= nodeMaxRetries) {
-          if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
-          continue; // no error recorded yet
-        }
-        keepRetrying = false;
-        // FINAL failure path
-        const deps = [...(fwd.get(node.id) || [])];
-        const suffix = deps.length
-          ? `. This affects downstream nodes: ${deps.join(', ')}`
-          : '';
-        const nodeErr = new Error(
-          `Node ${node.id} execution failed: ${e.message}${suffix}`,
-        );
-        nodeErr.nodeId = node.id;
-        nodeErr.timestamp = Date.now();
-        nodeErr.cause = e;
-
-        // Handle non-critical failures with warnings and no result nullification
-        if (isNonCritical || (continueOnError && !requiredIds.has(node.id))) {
-          console.warn(`Non-critical node failure: ${nodeErr.message}`);
-          errors.set(node.id, nodeErr);
-          // Don't set result for failed optional nodes - return special symbol so they don't appear in results
-          return Symbol('failed-optional-node');
-        }
-        errors.set(node.id, nodeErr);
-        throw nodeErr;
-      }
-    }
+    return this.scheduler.executeNode(node, context);
   }
 
   /**
@@ -678,59 +387,7 @@ class DAG {
    * @returns {Array} - Array of warnings (empty if no issues)
    */
   validateTopology(_options = {}) {
-    const { strict = true } = _options;
-    const warnings = [];
-
-    // Check for empty DAG
-    if (this.nodes.size === 0) {
-      throw new Error('DAG cannot be empty');
-    }
-
-    // Check for self-loops
-    for (const node of this.nodes.values()) {
-      if (node.outputs.includes(node)) {
-        throw new Error('Self-loop detected');
-      }
-    }
-
-    // Check for cycles by attempting topological sort
-    try {
-      this.topoSort();
-    } catch (error) {
-      if (error.message.includes('Cycle detected')) {
-        // Extract cycle information from error message or use error.cycle if available
-        let nodes = error.cycle;
-        if (!nodes && error.message.includes('involving node:')) {
-          // Parse cycle from message if cycle array not available
-          const match = error.message.match(/involving node: (.+)/);
-          if (match) {
-            nodes = match[1].split(' -> ');
-          }
-        }
-        if (!nodes) {
-          nodes = ['unknown'];
-        }
-
-        const err = new Error('Cycle detected in DAG');
-        err.cycle = nodes;
-        throw err;
-      }
-      throw error;
-    }
-
-    // Check for isolated nodes
-    for (const node of this.nodes.values()) {
-      if (node.inputs.length === 0 && node.outputs.length === 0) {
-        const message = `Orphaned node detected: ${node.id}`;
-        if (strict) {
-          throw new Error(message);
-        } else {
-          warnings.push(message);
-        }
-      }
-    }
-
-    return warnings;
+    return validateTopology(this.nodes, _options);
   }
 
   /**
@@ -739,62 +396,7 @@ class DAG {
    * @throws {Error} If cycles are detected
    */
   topoSort() {
-    const order = [];
-    const visited = new Set();
-    const visiting = new Set(); // Track nodes currently being visited
-
-    const visit = (node, path = []) => {
-      if (visiting.has(node.id)) {
-        const cycleStart = path.indexOf(node.id);
-        let cyclePath =
-          cycleStart >= 0
-            ? path.slice(cycleStart).concat([node.id])
-            : [node.id];
-        // Reverse interior to show forward traversal order (A->B->C->A)
-        if (cyclePath.length > 2) {
-          const startNode = cyclePath[0];
-          const restNodes = cyclePath.slice(1, -1).reverse();
-          cyclePath = [startNode, ...restNodes, startNode];
-        }
-        const cyclePathString = cyclePath.join(' -> ');
-        const error = new Error(
-          `Cycle detected involving node: ${cyclePathString}`,
-        );
-        error.cycle = cyclePath; // Array format for test assertions
-        throw error;
-      }
-
-      if (visited.has(node.id)) return;
-
-      visiting.add(node.id);
-      const newPath = [...path, node.id];
-
-      try {
-        node.inputs.forEach((input) => visit(input, newPath));
-      } catch (error) {
-        if (error.message.includes('Cycle detected')) {
-          throw error; // Preserve original message and cycle property
-        }
-        throw error;
-      }
-
-      visiting.delete(node.id);
-      visited.add(node.id);
-      order.push(node);
-    };
-
-    try {
-      for (const node of this.nodes.values()) {
-        visit(node);
-      }
-    } catch (error) {
-      if (error.message.includes('Cycle detected')) {
-        throw error;
-      }
-      throw new Error(`DAG topological sort failed: ${error.message}`);
-    }
-
-    return order;
+    return topologicalSort(this.nodes);
   }
 
   /**

package/src/dev/README.md

@@ -0,0 +1,126 @@
+# Development Tools
+
+This directory contains development tools for the RAG Pipeline Utils library.
+
+## Available Tools
+
+### Hot Reload Manager (`hot-reload.js`)
+
+Provides automatic reloading of plugins and configuration files during development.
+
+**Features:**
+
+- File watching with chokidar
+- Plugin hot-reloading with module cache clearing
+- Configuration hot-reloading with validation
+- State preservation and restoration
+- Lifecycle hooks (beforeReload, afterReload, onError, etc.)
+- Statistics tracking and history
+- Error handling with notifications
+
+**Usage:**
+
+```javascript
+const { createHotReloadManager } = require("./hot-reload");
+
+const hotReload = createHotReloadManager({
+  watchPaths: ["./src/plugins"],
+  configPath: ".ragrc.json",
+  verbose: true,
+});
+
+await hotReload.start();
+```
+
+### Development Server (`dev-server.js`)
+
+Integrated development server with hot reload support.
+
+**Features:**
+
+- Hot reload integration
+- Real-time notifications
+- Statistics dashboard
+- Event-driven architecture
+- Graceful shutdown
+- Pipeline integration
+
+**Usage:**
+
+```javascript
+const { createDevServer } = require("./dev-server");
+
+const devServer = createDevServer({
+  port: 3000,
+  watchPaths: ["./src/plugins"],
+  autoReload: true,
+});
+
+await devServer.start();
+```
+
+## Examples
+
+See `examples/hot-reload-usage.js` for comprehensive examples including:
+
+1. Basic hot reload setup
+2. Integration with plugin registry
+3. Development server usage
+4. Pipeline integration
+5. Custom state preservation
+6. Manual reload triggering
+7. Configuration validation
+8. Complete development workflows
+
+## Documentation
+
+Full documentation is available at `docs/hot-reload.md`.
+
+## Testing
+
+Tests are located in `__tests__/dev/hot-reload.test.js`.
+
+Run tests:
+
+```bash
+npm test -- __tests__/dev/hot-reload.test.js
+```
+
+## API Reference
+
+### Hot Reload Manager
+
+- `start()` - Start watching for changes
+- `stop()` - Stop watching
+- `triggerReload(filePath, type)` - Manually trigger reload
+- `registerHook(hookName, callback)` - Register lifecycle hook
+- `getStats()` - Get reload statistics
+- `getHistory()` - Get reload history
+
+### Development Server
+
+- `start()` - Start dev server
+- `stop()` - Stop dev server
+- `getStatus()` - Get server status
+- `getStatistics()` - Get formatted statistics
+- `triggerReload(filePath, type)` - Manually trigger reload
+- `registerHook(hookName, callback)` - Register lifecycle hook
+
+## Events
+
+Both tools emit various events:
+
+- `started` / `stopped`
+- `plugin:reloaded` / `plugin:reload-failed`
+- `config:reloaded` / `config:reload-failed`
+- `plugin:added` / `plugin:removed`
+- `error`
+- `notification` (DevServer only)
+
+## Dependencies
+
+- `chokidar` - File system watching
+
+## License
+
+GPL-3.0