@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0

package/dist/types/dag/core/execution-scheduler.d.ts

@@ -0,0 +1,191 @@
+export = ExecutionScheduler;
+/**
+ * ExecutionScheduler class for managing DAG node execution
+ *
+ * Provides stateless execution scheduling with:
+ * - Concurrency control via semaphore
+ * - Timeout enforcement
+ * - Retry logic with exponential backoff
+ * - Dependency resolution
+ * - Error handling and reporting
+ *
+ * @class
+ * @since 2.2.3
+ */
+declare class ExecutionScheduler {
+    /**
+     * Create an ExecutionScheduler instance
+     *
+     * @param {Object} options - Scheduler configuration
+     * @param {Function} options.getSinkIds - Function to get sink node IDs
+     * @param {Object} [options.logger] - Logger instance for debugging
+     */
+    constructor(options?: {
+        getSinkIds: Function;
+        logger?: any;
+    });
+    getSinkIds: Function;
+    logger: any;
+    errorContext: ErrorContext;
+    /**
+     * Schedule and execute all nodes with concurrency management
+     *
+     * Executes nodes in topological order, respecting dependencies and
+     * managing concurrency via semaphore. Each node is executed only after
+     * all its dependencies have completed or failed.
+     *
+     * @param {DAGNode[]} order - Topologically sorted array of nodes
+     * @param {Object} context - Execution context
+     * @param {number} [context.concurrency] - Maximum concurrent executions
+     * @param {Map} context.results - Results map (node ID -> result)
+     * @param {Map} context.errors - Errors map (node ID -> error)
+     * @param {Map} context.fwd - Forward adjacency map
+     * @param {boolean} [context.continueOnError] - Continue on non-critical errors
+     * @param {Set} context.requiredIds - Set of required node IDs
+     * @param {any} context.seed - Initial seed value
+     * @param {boolean} [context.gracefulDegradation] - Enable graceful degradation
+     * @param {string[]} [context.requiredNodes] - Array of required node IDs
+     * @param {boolean} [context.retryFailedNodes] - Enable retry for failed nodes
+     * @param {number} [context.maxRetries] - Maximum retry attempts
+     * @returns {Promise<void>}
+     *
+     * @example
+     * const scheduler = new ExecutionScheduler({ getSinkIds });
+     * await scheduler.scheduleExecution(sortedNodes, {
+     *   concurrency: 5,
+     *   results: new Map(),
+     *   errors: new Map(),
+     *   fwd: adjacencyMap,
+     *   requiredIds: new Set(['critical-node']),
+     *   seed: initialInput
+     * });
+     *
+     * @performance O(V) where V = number of nodes
+     * @since 2.2.3
+     */
+    scheduleExecution(order: DAGNode[], context: {
+        concurrency?: number;
+        results: Map<any, any>;
+        errors: Map<any, any>;
+        fwd: Map<any, any>;
+        continueOnError?: boolean;
+        requiredIds: Set<any>;
+        seed: any;
+        gracefulDegradation?: boolean;
+        requiredNodes?: string[];
+        retryFailedNodes?: boolean;
+        maxRetries?: number;
+    }): Promise<void>;
+    /**
+     * Execute a single DAG node with retry logic and error handling
+     *
+     * Executes node.run() with comprehensive error handling:
+     * - Validates node.run is a function
+     * - Determines node criticality
+     * - Prepares input from dependencies
+     * - Implements retry logic with configurable attempts
+     * - Reports errors with downstream impact analysis
+     *
+     * @param {DAGNode} node - Node to execute
+     * @param {Object} context - Execution context
+     * @param {Map} context.results - Results map
+     * @param {Map} context.errors - Errors map
+     * @param {Map} context.fwd - Forward adjacency map
+     * @param {boolean} [context.continueOnError] - Continue on non-critical errors
+     * @param {Set} context.requiredIds - Required node IDs
+     * @param {any} context.seed - Seed value for source nodes
+     * @param {boolean} [context.gracefulDegradation] - Enable graceful degradation
+     * @param {string[]} [context.requiredNodes] - Required node IDs array
+     * @param {boolean} [context.retryFailedNodes] - Enable retry
+     * @param {number} [context.maxRetries] - Max retry attempts
+     * @returns {Promise<any>} Node execution result or Symbol('failed-optional-node')
+     *
+     * @throws {Error} Node execution error (if node is critical)
+     *
+     * @example
+     * const result = await scheduler.executeNode(myNode, {
+     *   results: new Map(),
+     *   errors: new Map(),
+     *   fwd: adjacencyMap,
+     *   requiredIds: new Set(),
+     *   seed: initialData,
+     *   retryFailedNodes: true,
+     *   maxRetries: 3
+     * });
+     *
+     * @performance O(1) for single execution, O(k) for k retry attempts
+     * @complexity Cyclomatic complexity: 12 (acceptable for critical execution path)
+     * @since 2.2.3
+     */
+    executeNode(node: DAGNode, context: {
+        results: Map<any, any>;
+        errors: Map<any, any>;
+        fwd: Map<any, any>;
+        continueOnError?: boolean;
+        requiredIds: Set<any>;
+        seed: any;
+        gracefulDegradation?: boolean;
+        requiredNodes?: string[];
+        retryFailedNodes?: boolean;
+        maxRetries?: number;
+    }): Promise<any>;
+    /**
+     * Execute function with optional timeout enforcement
+     *
+     * Uses Promise.race to enforce execution timeout. If timeout is not specified
+     * or is <= 0, executes without timeout constraint.
+     *
+     * @param {Function} runFn - Async function to execute
+     * @param {number} [timeout] - Timeout in milliseconds (0 or undefined = no timeout)
+     * @returns {Promise<void>}
+     *
+     * @throws {Error} 'Execution timeout' if execution exceeds timeout
+     *
+     * @example
+     * await scheduler.executeWithTimeout(
+     *   () => executeLongRunningTask(),
+     *   30000 // 30 second timeout
+     * );
+     *
+     * @performance O(1) overhead for timeout setup
+     * @since 2.2.3
+     */
+    executeWithTimeout(runFn: Function, timeout?: number): Promise<void>;
+    /**
+     * Check if a node can be executed based on dependency satisfaction
+     *
+     * A node can execute if all its input dependencies have either:
+     * - Completed successfully (have a result)
+     * - Failed (have an error)
+     * - Are optional (graceful degradation enabled and not in required nodes)
+     *
+     * @param {DAGNode} node - Node to check
+     * @param {Object} context - Execution context
+     * @param {Map} context.results - Results map
+     * @param {Map} context.errors - Errors map
+     * @param {boolean} [context.gracefulDegradation] - Enable graceful degradation
+     * @param {string[]} [context.requiredNodes] - Required node IDs
+     * @returns {boolean} True if node can execute, false otherwise
+     *
+     * @example
+     * const canRun = scheduler.canExecuteNode(myNode, {
+     *   results: new Map([['dep1', result1]]),
+     *   errors: new Map(),
+     *   gracefulDegradation: true,
+     *   requiredNodes: ['critical-dep']
+     * });
+     *
+     * @performance O(d) where d = number of input dependencies
+     * @since 2.2.3
+     */
+    canExecuteNode(node: DAGNode, context: {
+        results: Map<any, any>;
+        errors: Map<any, any>;
+        gracefulDegradation?: boolean;
+        requiredNodes?: string[];
+    }): boolean;
+}
+declare namespace ExecutionScheduler {
+    export { ExecutionScheduler as default };
+}
+import ErrorContext = require("./error-context.js");

package/dist/types/dag/core/topology-validation.d.ts

@@ -0,0 +1,226 @@
+/**
+ * @fileoverview DAG Topology Validation Module
+ *
+ * Provides topology validation utilities for Directed Acyclic Graphs (DAGs),
+ * including cycle detection, topological sorting, and structural validation.
+ *
+ * @module dag/core/topology-validation
+ * @author Ali Kahwaji
+ * @since 2.2.2
+ * @version 1.0.0
+ */
+/**
+ * Builds forward and reverse adjacency maps for a DAG
+ *
+ * Creates two Maps:
+ * - Forward: Maps each node ID to a Set of its children (outgoing edges)
+ * - Reverse: Maps each node ID to a Set of its parents (incoming edges)
+ *
+ * @param {Map<string, DAGNode>} nodes - Map of node IDs to DAGNode instances
+ * @returns {{fwd: Map<string, Set<string>>, rev: Map<string, Set<string>>}}
+ *   Adjacency maps for forward and reverse traversal
+ *
+ * @example
+ * const nodes = new Map([
+ *   ['A', nodeA],
+ *   ['B', nodeB],
+ *   ['C', nodeC]
+ * ]);
+ * const { fwd, rev } = buildAdjacency(nodes);
+ * // fwd.get('A') => Set(['B', 'C']) - A points to B and C
+ * // rev.get('B') => Set(['A']) - B is pointed to by A
+ *
+ * @performance O(V + E) where V = number of nodes, E = number of edges
+ * @since 2.2.2
+ */
+export function buildAdjacency(nodes: Map<string, DAGNode>): {
+    fwd: Map<string, Set<string>>;
+    rev: Map<string, Set<string>>;
+};
+/**
+ * Gets sink node IDs (nodes with no outgoing edges)
+ *
+ * Sink nodes are terminal nodes in the DAG that have no children.
+ * They represent final outputs of the pipeline.
+ *
+ * @param {Map<string, Set<string>>} fwd - Forward adjacency map
+ * @returns {string[]} Array of sink node IDs
+ *
+ * @example
+ * const fwd = new Map([
+ *   ['A', new Set(['B'])],
+ *   ['B', new Set()],      // B is a sink
+ *   ['C', new Set()]       // C is a sink
+ * ]);
+ * const sinks = getSinkIds(fwd);
+ * console.log(sinks); // ['B', 'C']
+ *
+ * @performance O(V) where V = number of nodes
+ * @since 2.2.2
+ */
+export function getSinkIds(fwd: Map<string, Set<string>>): string[];
+/**
+ * Performs topological sort with comprehensive cycle detection
+ *
+ * Uses depth-first search (DFS) with path tracking to detect cycles.
+ * If a cycle is detected, the error includes the complete cycle path.
+ *
+ * @param {Map<string, DAGNode>} nodes - Map of node IDs to DAGNode instances
+ * @returns {DAGNode[]} Topologically sorted array of nodes
+ *
+ * @throws {Error} If a cycle is detected. Error includes:
+ *   - message: Description with cycle path (e.g., "Cycle detected involving node: A -> B -> C -> A")
+ *   - cycle: Array of node IDs forming the cycle
+ * @throws {Error} If topological sort fails for other reasons
+ *
+ * @example
+ * // Valid DAG
+ * const nodes = new Map([
+ *   ['A', nodeA], // A -> B
+ *   ['B', nodeB], // B -> C
+ *   ['C', nodeC]  // C (sink)
+ * ]);
+ * const sorted = topologicalSort(nodes);
+ * console.log(sorted.map(n => n.id)); // ['C', 'B', 'A'] (reverse dependency order)
+ *
+ * @example
+ * // Cyclic DAG (throws error)
+ * const nodes = new Map([
+ *   ['A', nodeA], // A -> B
+ *   ['B', nodeB], // B -> C
+ *   ['C', nodeC]  // C -> A (creates cycle)
+ * ]);
+ * try {
+ *   topologicalSort(nodes);
+ * } catch (error) {
+ *   console.log(error.message); // "Cycle detected involving node: A -> B -> C -> A"
+ *   console.log(error.cycle);   // ['A', 'B', 'C', 'A']
+ * }
+ *
+ * @algorithm
+ * 1. Initialize empty order array, visited set, and visiting set
+ * 2. For each unvisited node, perform DFS:
+ *    a. If node is currently being visited (in visiting set), cycle detected
+ *    b. Mark node as visiting
+ *    c. Recursively visit all input nodes (dependencies)
+ *    d. Mark node as visited, remove from visiting
+ *    e. Add node to order array
+ * 3. Return order array (nodes in reverse topological order)
+ *
+ * @performance O(V + E) where V = number of nodes, E = number of edges
+ * @complexity Cyclomatic complexity: 6 (within acceptable limits)
+ * @since 2.2.2
+ */
+export function topologicalSort(nodes: Map<string, DAGNode>): DAGNode[];
+/**
+ * Validates basic DAG structure
+ *
+ * Performs fundamental validation checks:
+ * - DAG is not empty
+ * - No cycles exist (via topological sort)
+ * - At least one sink node exists
+ *
+ * This is the primary validation method called before DAG execution.
+ *
+ * @param {Map<string, DAGNode>} nodes - Map of node IDs to DAGNode instances
+ * @returns {boolean} True if validation passes
+ *
+ * @throws {Error} If DAG is empty - "DAG is empty - no nodes to execute"
+ * @throws {Error} If cycle detected - "DAG validation failed: DAG topological sort failed: Cycle detected involving node: ..."
+ *   - error.cycle: Array of node IDs forming the cycle
+ * @throws {Error} If no sink nodes - "DAG has no sink nodes - no final output available"
+ * @throws {Error} For other validation failures
+ *
+ * @example
+ * // Valid DAG
+ * const nodes = new Map([
+ *   ['load', loadNode],
+ *   ['process', processNode],
+ *   ['save', saveNode]
+ * ]);
+ * const isValid = validateDAG(nodes); // true
+ *
+ * @example
+ * // Empty DAG
+ * try {
+ *   validateDAG(new Map());
+ * } catch (error) {
+ *   console.log(error.message); // "DAG is empty - no nodes to execute"
+ * }
+ *
+ * @example
+ * // No sink nodes (all nodes have outputs)
+ * const nodes = new Map([
+ *   ['A', nodeA] // A -> B
+ * ]);
+ * // Assuming all nodes have outputs
+ * try {
+ *   validateDAG(nodes);
+ * } catch (error) {
+ *   console.log(error.message); // "DAG has no sink nodes - no final output available"
+ * }
+ *
+ * @performance O(V + E) due to topological sort
+ * @since 2.2.2
+ */
+export function validateDAG(nodes: Map<string, DAGNode>): boolean;
+/**
+ * Validates DAG topology with advanced options and warnings
+ *
+ * Performs comprehensive validation including:
+ * - Empty DAG check
+ * - Self-loop detection
+ * - Cycle detection via topological sort
+ * - Isolated node detection (nodes with no connections)
+ *
+ * Supports strict and non-strict modes. In non-strict mode, some issues
+ * (like isolated nodes) generate warnings instead of throwing errors.
+ *
+ * @param {Map<string, DAGNode>} nodes - Map of node IDs to DAGNode instances
+ * @param {Object} options - Validation options
+ * @param {boolean} [options.strict=true] - If true, throw errors for all issues.
+ *   If false, return warnings for non-critical issues
+ * @returns {string[]} Array of warning messages (empty if no warnings in strict mode)
+ *
+ * @throws {Error} If DAG is empty - "DAG cannot be empty"
+ * @throws {Error} If self-loop detected - "Self-loop detected"
+ * @throws {Error} If cycle detected - "Cycle detected in DAG"
+ *   - error.cycle: Array of node IDs forming the cycle
+ * @throws {Error} If orphaned node found in strict mode
+ *
+ * @example
+ * // Strict mode (default)
+ * const nodes = new Map([['isolated', isolatedNode]]);
+ * try {
+ *   validateTopology(nodes, { strict: true });
+ * } catch (error) {
+ *   console.log(error.message); // "Orphaned node detected: isolated"
+ * }
+ *
+ * @example
+ * // Non-strict mode (warnings)
+ * const nodes = new Map([
+ *   ['A', nodeA],
+ *   ['isolated', isolatedNode]
+ * ]);
+ * const warnings = validateTopology(nodes, { strict: false });
+ * console.log(warnings); // ["Orphaned node detected: isolated"]
+ *
+ * @example
+ * // Self-loop detection
+ * const nodeWithSelfLoop = new DAGNode('X', () => {});
+ * nodeWithSelfLoop.outputs.push(nodeWithSelfLoop);
+ * const nodes = new Map([['X', nodeWithSelfLoop]]);
+ * try {
+ *   validateTopology(nodes);
+ * } catch (error) {
+ *   console.log(error.message); // "Self-loop detected"
+ * }
+ *
+ * @performance O(V + E) due to topological sort and graph traversal
+ * @complexity Cyclomatic complexity: 5 (within acceptable limits)
+ * @since 2.2.2
+ */
+export function validateTopology(nodes: Map<string, DAGNode>, options?: {
+    strict?: boolean;
+}): string[];

package/dist/types/dag/dag-engine.d.ts

@@ -0,0 +1,156 @@
+export class DAG {
+    nodes: Map<any, any>;
+    scheduler: ExecutionScheduler;
+    errorContext: ErrorContext;
+    addNode(id: any, _fn: any, options?: {}): DAGNode;
+    connect(fromId: any, toId: any): void;
+    /**
+     * Parse and normalize execute parameters
+     * @param {any} seedOrOptions - Seed data or options object
+     * @param {Object} _options - Additional options (if first param is seed)
+     * @returns {Object} - Normalized {seed, options} object
+     */
+    _parseExecuteParams(seedOrOptions: any, _options?: any): any;
+    /**
+     * Execute all nodes in topological order with concurrency control
+     * @param {Array} order - Topologically sorted nodes
+     * @param {Object} context - Execution context
+     * @returns {Promise<void>}
+     */
+    _runAllNodes(order: any[], context: any): Promise<void>;
+    /**
+     * Check if a node can be executed based on dependency satisfaction
+     * @param {DAGNode} node - Node to check
+     * @param {Object} context - Execution context
+     * @returns {boolean}
+     */
+    _canExecuteNode(node: DAGNode, context: any): boolean;
+    /**
+     * Initialize execution state and validate DAG
+     * @returns {Object} - Execution state object
+     */
+    _initializeExecutionState(): any;
+    /**
+     * Handle execution errors with appropriate wrapping
+     * @param {Error} error - Error to handle
+     * @throws {Error} - Appropriately wrapped error
+     */
+    _handleExecutionError(error: Error): void;
+    /**
+     * Execute function with optional timeout
+     * @param {Function} runFn - Function to execute
+     * @param {number} timeout - Timeout in milliseconds
+     * @returns {Promise<void>}
+     */
+    _executeWithTimeout(runFn: Function, timeout: number): Promise<void>;
+    /**
+     * Process execution results and handle checkpointing
+     * @param {Map} results - Node execution results
+     * @param {Map} errors - Node execution errors
+     * @param {Map} fwd - Forward adjacency map
+     * @param {Object} options - Processing options
+     * @returns {any} - Processed results
+     */
+    _processResults(results: Map<any, any>, errors: Map<any, any>, fwd: Map<any, any>, options: any): any;
+    /**
+     * Executes the DAG with optional seed data and configuration
+     * @param {any} seedOrOptions - Seed data or options object
+     * @param {Object} _options - Additional options (if first param is seed)
+     * @returns {Promise<any>} - Execution results
+     */
+    execute(seedOrOptions: any, _options?: any): Promise<any>;
+    /**
+     * Helper for running with optional global timeout
+     * @param {Function} runFn - Function to run
+     * @param {number} timeout - Timeout in milliseconds
+     * @returns {Promise} - Promise that resolves/rejects with timeout handling
+     */
+    _runWithOptionalGlobalTimeout(runFn: Function, timeout: number): Promise<any>;
+    /**
+     * Build adjacency lists for the DAG
+     * @returns {Object} - Object with fwd and rev adjacency maps
+     */
+    _buildAdjacency(): any;
+    /**
+     * Get sink node IDs (nodes with no outgoing edges)
+     * @param {Map} fwd - Forward adjacency map
+     * @returns {Array} - Array of sink node IDs
+     */
+    _getSinkIds(fwd: Map<any, any>): any[];
+    /**
+     * Get required node IDs (all sinks + all their ancestors)
+     * @param {Map} fwd - Forward adjacency map
+     * @param {Map} rev - Reverse adjacency map
+     * @returns {Set} - Set of required node IDs
+     */
+    _getRequiredIds(fwd: Map<any, any>, rev: Map<any, any>): Set<any>;
+    /**
+     * Build adjacency list for the DAG (legacy method for compatibility)
+     * @returns {Map} - Adjacency list mapping node IDs to arrays of dependent node IDs
+     */
+    buildAdjacencyList(): Map<any, any>;
+    /**
+     * Validates the DAG structure
+     * @returns {boolean} - True if valid
+     * @throws {Error} - If validation fails
+     */
+    validate(): boolean;
+    /**
+     * Execute a single node with input data
+     * @param {DAGNode} node - Node to execute
+     * @param {Object} context - Execution context
+     * @returns {Promise<any>} Node execution result
+     */
+    executeNode(node: DAGNode, context: any): Promise<any>;
+    /**
+     * Validate topology with advanced options
+     * @param {Object} _options - Validation options
+     * @returns {Array} - Array of warnings (empty if no issues)
+     */
+    validateTopology(_options?: any): any[];
+    /**
+     * Performs topological sort with cycle detection
+     * @returns {DAGNode[]} Topologically sorted nodes
+     * @throws {Error} If cycles are detected
+     */
+    topoSort(): DAGNode[];
+    /**
+     * Resume execution from checkpoint data
+     * @param {Object} checkpointData - Checkpoint data to resume from
+     * @returns {Promise<Map>} - Results Map from resumed execution
+     */
+    resume(checkpointData?: any): Promise<Map<any, any>>;
+    /**
+     * Save execution checkpoint
+     * @param {string} checkpointId - Unique checkpoint identifier
+     * @param {Object} state - Execution state to save
+     */
+    saveCheckpoint(checkpointId: string, state: any): void;
+    checkpoints: Map<any, any>;
+    /**
+     * Load execution checkpoint
+     * @param {string} checkpointId - Unique checkpoint identifier
+     * @returns {Object|null} - Saved execution state or null if not found
+     */
+    loadCheckpoint(checkpointId: string): any | null;
+    /**
+     * List all available checkpoints
+     * @returns {Array} - Array of checkpoint IDs with metadata
+     */
+    listCheckpoints(): any[];
+    /**
+     * Clear execution checkpoint
+     * @param {string} checkpointId - Unique checkpoint identifier
+     */
+    clearCheckpoint(checkpointId: string): void;
+    /**
+     * Get all downstream nodes that depend on the given node
+     * @param {DAGNode} node - Node to find downstream dependencies for
+     * @returns {DAGNode[]} Array of downstream nodes
+     */
+    getDownstreamNodes(node: DAGNode): DAGNode[];
+}
+import DAGNode = require("./dag-node.js");
+import ExecutionScheduler = require("./core/execution-scheduler.js");
+import ErrorContext = require("./core/error-context.js");
+export { DAGNode };

package/dist/types/dag/dag-node.d.ts

@@ -0,0 +1,37 @@
+export = DAGNode;
+/**
+ * Version: 2.0.2
+ * Path: /src/dag/dag-node.js
+ * Description: DAG node class for representing individual nodes in the DAG
+ * Author: Ali Kahwaji
+ */
+declare class DAGNode {
+    constructor(id: any, run: any, options?: {});
+    id: any;
+    run: any;
+    inputs: any[];
+    outputs: any[];
+    timeout: any;
+    retries: any;
+    priority: any;
+    /**
+     * Add an output connection to another node
+     * @param {DAGNode} node - Target node to connect to
+     */
+    addOutput(node: DAGNode): this;
+    /**
+     * Add an input connection from another node
+     * @param {DAGNode} node - Source node to connect from
+     */
+    addInput(node: DAGNode): this;
+    /**
+     * Remove an output connection
+     * @param {DAGNode} node - Target node to disconnect from
+     */
+    removeOutput(node: DAGNode): this;
+    /**
+     * Remove an input connection
+     * @param {DAGNode} node - Source node to disconnect from
+     */
+    removeInput(node: DAGNode): this;
+}