@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. package/.ragrc.schema.json +83 -192
  2. package/CHANGELOG.md +345 -0
  3. package/README.md +418 -117
  4. package/SECURITY.md +5 -5
  5. package/dist/experimental.cjs +2 -0
  6. package/dist/experimental.d.ts +1 -0
  7. package/dist/experimental.mjs +5 -0
  8. package/dist/index.cjs +4992 -1242
  9. package/dist/index.d.ts +80 -2
  10. package/dist/index.mjs +1 -1
  11. package/dist/types/agentic/agentic-pipeline.d.ts +103 -0
  12. package/dist/types/agentic/iterative-retriever.d.ts +85 -0
  13. package/dist/types/agentic/query-planner.d.ts +145 -0
  14. package/dist/types/agentic/self-critique.d.ts +45 -0
  15. package/dist/types/ai/adaptive-retrieval.d.ts +44 -0
  16. package/dist/types/ai/federation/federated-learning-coordinator.d.ts +62 -0
  17. package/dist/types/ai/multimodal-processing.d.ts +32 -0
  18. package/dist/types/ai/multimodal.d.ts +39 -0
  19. package/dist/types/ai/retrieval-engine.d.ts +37 -0
  20. package/dist/types/ai/training/model-training-orchestrator.d.ts +31 -0
  21. package/dist/types/chunking/chunking-engine.d.ts +70 -0
  22. package/dist/types/chunking/strategies/fixed-size.d.ts +17 -0
  23. package/dist/types/chunking/strategies/recursive.d.ts +20 -0
  24. package/dist/types/chunking/strategies/semantic.d.ts +19 -0
  25. package/dist/types/chunking/strategies/sentence.d.ts +13 -0
  26. package/dist/types/chunking/strategies/structure-aware.d.ts +19 -0
  27. package/dist/types/citation/citation-tracker.d.ts +64 -0
  28. package/dist/types/citation/hallucination-detector.d.ts +40 -0
  29. package/dist/types/citation/source-mapper.d.ts +37 -0
  30. package/dist/types/connectors/anthropic-connector.d.ts +92 -0
  31. package/dist/types/connectors/base-connector.d.ts +87 -0
  32. package/dist/types/connectors/cohere-connector.d.ts +89 -0
  33. package/dist/types/connectors/connector-utils.d.ts +110 -0
  34. package/dist/types/connectors/local-embedder.d.ts +86 -0
  35. package/dist/types/connectors/memory-retriever.d.ts +91 -0
  36. package/dist/types/connectors/ollama-connector.d.ts +110 -0
  37. package/dist/types/connectors/openai-connector.d.ts +114 -0
  38. package/dist/types/connectors/rag-prompt.d.ts +40 -0
  39. package/dist/types/cost/cost-calculator.d.ts +152 -0
  40. package/dist/types/cost/cost-tracker.d.ts +126 -0
  41. package/dist/types/cost/token-budget.d.ts +110 -0
  42. package/dist/types/dag/core/error-context.d.ts +239 -0
  43. package/dist/types/dag/core/execution-scheduler.d.ts +191 -0
  44. package/dist/types/dag/core/topology-validation.d.ts +226 -0
  45. package/dist/types/dag/dag-engine.d.ts +156 -0
  46. package/dist/types/dag/dag-node.d.ts +37 -0
  47. package/dist/types/dashboard/dashboard-generator.d.ts +127 -0
  48. package/dist/types/dashboard/metrics-aggregator.d.ts +218 -0
  49. package/dist/types/debug/execution-tracer.d.ts +57 -0
  50. package/dist/types/debug/trace-inspector.d.ts +31 -0
  51. package/dist/types/ecosystem/plugin-certification.d.ts +169 -0
  52. package/dist/types/enterprise/data-governance.d.ts +203 -0
  53. package/dist/types/enterprise/multi-tenancy.d.ts +137 -0
  54. package/dist/types/enterprise/sso-integration.d.ts +420 -0
  55. package/dist/types/evaluate/context-metrics.d.ts +55 -0
  56. package/dist/types/evaluate/faithfulness.d.ts +41 -0
  57. package/dist/types/evaluate/groundedness.d.ts +60 -0
  58. package/dist/types/evaluate/pipeline-evaluator.d.ts +88 -0
  59. package/dist/types/evaluate/relevance.d.ts +21 -0
  60. package/dist/types/evaluate/scoring.d.ts +123 -0
  61. package/dist/types/experimental.d.ts +15 -0
  62. package/dist/types/graph/entity-extractor.d.ts +84 -0
  63. package/dist/types/graph/graph-index.d.ts +49 -0
  64. package/dist/types/graph/graph-retriever.d.ts +81 -0
  65. package/dist/types/graph/knowledge-graph.d.ts +129 -0
  66. package/dist/types/guardrails/guardrails-pipeline.d.ts +63 -0
  67. package/dist/types/guardrails/injection-detector.d.ts +47 -0
  68. package/dist/types/guardrails/post-generation-guard.d.ts +61 -0
  69. package/dist/types/guardrails/pre-retrieval-guard.d.ts +64 -0
  70. package/dist/types/guardrails/retrieval-guard.d.ts +57 -0
  71. package/dist/types/mcp/mcp-server.d.ts +85 -0
  72. package/dist/types/mcp/mcp-tool-builder.d.ts +55 -0
  73. package/dist/types/reranker/cascade-reranker.d.ts +40 -0
  74. package/dist/types/reranker/embedding-reranker.d.ts +77 -0
  75. package/dist/types/reranker/scoring-reranker.d.ts +41 -0
  76. package/dist/types/retrieval/bm25-search.d.ts +43 -0
  77. package/dist/types/retrieval/file-vector-store.d.ts +87 -0
  78. package/dist/types/retrieval/hybrid-retriever.d.ts +35 -0
  79. package/dist/types/retrieval/rank-fusion.d.ts +10 -0
  80. package/dist/types/security/jwt-config.d.ts +80 -0
  81. package/dist/types/security/jwt-validator.d.ts +195 -0
  82. package/dist/types/security/plugin-sandbox.d.ts +158 -0
  83. package/dist/types/security/plugin-signature-verifier.d.ts +139 -0
  84. package/dist/types/security/validator.d.ts +5 -0
  85. package/dist/types/streaming/embedding-stream.d.ts +65 -0
  86. package/dist/types/streaming/sse-adapter.d.ts +59 -0
  87. package/dist/types/streaming/stream-router.d.ts +88 -0
  88. package/dist/types/streaming/streaming-embedder.d.ts +99 -0
  89. package/dist/types/streaming/websocket-adapter.d.ts +71 -0
  90. package/dist/types/templates/scaffold.d.ts +55 -0
  91. package/dist/types/templates/template-registry.d.ts +63 -0
  92. package/dist/types/utils/embed-query.d.ts +20 -0
  93. package/dist/types/utils/logger.d.ts +50 -0
  94. package/dist/types/utils/rate-limiter.d.ts +66 -0
  95. package/dist/types/utils/retry.d.ts +27 -0
  96. package/dist/types/utils/run-abort.d.ts +34 -0
  97. package/dist/types/utils/secure-logging.d.ts +51 -0
  98. package/dist/types/utils/structured-logger.d.ts +126 -0
  99. package/package.json +53 -29
  100. package/src/agentic/agentic-pipeline.js +75 -7
  101. package/src/agentic/iterative-retriever.js +47 -10
  102. package/src/agentic/query-planner.js +329 -78
  103. package/src/citation/citation-tracker.js +6 -4
  104. package/src/citation/hallucination-detector.js +2 -2
  105. package/src/citation/source-mapper.js +15 -28
  106. package/src/cli/commands/ai.js +101 -0
  107. package/src/cli/commands/plugin-hub/event-handlers.js +6 -0
  108. package/src/cli/commands/plugin-hub/hub-commands.js +0 -74
  109. package/src/cli/commands/plugin-hub/local-commands.js +143 -0
  110. package/src/cli/commands/plugin-hub.js +96 -36
  111. package/src/cli/diagnostics/config-checker.js +76 -63
  112. package/src/cli/diagnostics/environment-checker.js +62 -31
  113. package/src/cli/diagnostics/plugin-checker.js +36 -89
  114. package/src/cli/doctor-command.js +39 -5
  115. package/src/cli/enhanced-cli-commands.js +72 -217
  116. package/src/cli/handlers/ai-handler.js +364 -0
  117. package/src/cli/handlers/diagnostics-handler.js +78 -31
  118. package/src/cli/handlers/ingest-handler.js +28 -22
  119. package/src/cli/handlers/init-handler.js +148 -25
  120. package/src/cli/handlers/query-handler.js +41 -26
  121. package/src/cli/handlers/run-handler.js +16 -167
  122. package/src/cli/handlers/serve-handler.js +60 -0
  123. package/src/cli/services/bench-stats.js +54 -0
  124. package/src/cli/services/benchmark.js +161 -0
  125. package/src/cli/services/feedback.js +155 -0
  126. package/src/cli/services/rag-service.js +322 -0
  127. package/src/cli/shared/cli-utils.js +24 -7
  128. package/src/cli/shared/config-context.js +21 -0
  129. package/src/cli/wizard/config-builder.js +81 -0
  130. package/src/cli/wizard/init-wizard.js +617 -0
  131. package/src/config/component-factory.js +125 -0
  132. package/src/config/load-config.js +29 -10
  133. package/src/config/load-plugin-config.js +46 -2
  134. package/src/connectors/anthropic-connector.js +93 -55
  135. package/src/connectors/base-connector.js +74 -19
  136. package/src/connectors/cohere-connector.js +88 -32
  137. package/src/connectors/connector-utils.js +302 -0
  138. package/src/connectors/local-embedder.js +43 -5
  139. package/src/connectors/memory-retriever.js +108 -15
  140. package/src/connectors/ollama-connector.js +83 -49
  141. package/src/connectors/openai-connector.js +108 -44
  142. package/src/connectors/rag-prompt.js +88 -0
  143. package/src/core/create-pipeline.js +209 -60
  144. package/src/core/plugin-registry.js +87 -11
  145. package/src/cost/cost-calculator.js +205 -32
  146. package/src/dev/dev-server.js +166 -11
  147. package/src/dev/hot-reload.js +40 -1
  148. package/src/ecosystem/plugin-checks.js +181 -0
  149. package/src/ecosystem/plugin-hub.js +50 -125
  150. package/src/ecosystem/plugin-installer.js +298 -0
  151. package/src/ecosystem/plugin-probe.js +127 -0
  152. package/src/ecosystem/plugin-scanner.js +185 -0
  153. package/src/ecosystem/tar-extract.js +114 -0
  154. package/src/enterprise/audit-logging.js +158 -29
  155. package/src/enterprise/sso-integration.js +715 -228
  156. package/src/evaluate/context-metrics.js +28 -16
  157. package/src/evaluate/evaluator.js +3 -3
  158. package/src/evaluate/faithfulness.js +18 -14
  159. package/src/evaluate/groundedness.js +43 -12
  160. package/src/evaluate/pipeline-evaluator.js +35 -8
  161. package/src/evaluate/scoring.js +251 -3
  162. package/src/experimental.js +71 -0
  163. package/src/guardrails/guardrails-pipeline.js +69 -7
  164. package/src/guardrails/injection-detector.js +283 -0
  165. package/src/guardrails/post-generation-guard.js +19 -0
  166. package/src/guardrails/pre-retrieval-guard.js +36 -36
  167. package/src/guardrails/retrieval-guard.js +58 -8
  168. package/src/index.js +36 -1
  169. package/src/loader/document-reader.js +91 -0
  170. package/src/reranker/cascade-reranker.js +3 -0
  171. package/src/reranker/embedding-reranker.js +136 -21
  172. package/src/reranker/llm-reranker.js +79 -12
  173. package/src/reranker/scoring-reranker.js +3 -0
  174. package/src/retrieval/bm25-search.js +12 -6
  175. package/src/retrieval/file-vector-store.js +212 -0
  176. package/src/retrieval/hybrid-retriever.js +22 -19
  177. package/src/security/jwt-config.js +13 -2
  178. package/src/security/jwt-validator.js +134 -30
  179. package/src/security/plugin-sandbox.js +18 -1
  180. package/src/security/plugin-signature-verifier.js +245 -70
  181. package/src/utils/deprecate.js +69 -0
  182. package/src/utils/embed-query.js +33 -0
  183. package/src/utils/input-sanitizer.js +19 -4
  184. package/src/utils/logger.js +34 -4
  185. package/src/utils/run-abort.js +102 -0
  186. package/src/ai/federated-learning.js +0 -211
  187. package/src/ai/federation.js +0 -71
  188. package/src/ai/index.js +0 -22
  189. package/src/ai/model-trainer.js +0 -217
  190. package/src/ai/model-training.js +0 -202
  191. package/src/ai/multimodal/multi-modal-processor.js +0 -345
  192. package/src/ai/orchestrator.js +0 -65
  193. package/src/ai/retrieval/adaptive-retrieval-engine.js +0 -398
  194. package/src/cli/commands/ai-ml/adaptive-commands.js +0 -178
  195. package/src/cli/commands/ai-ml/federated-commands.js +0 -207
  196. package/src/cli/commands/ai-ml/index.js +0 -17
  197. package/src/cli/commands/ai-ml/multimodal-commands.js +0 -187
  198. package/src/cli/commands/ai-ml/shared.js +0 -51
  199. package/src/cli/commands/ai-ml/training-commands.js +0 -228
  200. package/src/cli/commands/ai-ml.js +0 -386
  201. package/src/cli/commands/docs.js +0 -228
  202. package/src/cli/commands/dx.js +0 -538
  203. package/src/cli/commands/wizard.js +0 -627
  204. package/src/cli/input-sanitization-middleware.js +0 -356
  205. package/src/cli/interactive-wizard.js +0 -373
  206. package/src/cli/templates/development.template.json +0 -64
  207. package/src/cli/templates/minimal.template.json +0 -33
  208. package/src/cli/templates/production.template.json +0 -104
  209. package/src/cli/templates/testing.template.json +0 -45
  210. package/src/cli/wizard/configuration.js +0 -148
  211. package/src/cli/wizard/observability.js +0 -79
  212. package/src/cli/wizard/performance.js +0 -76
  213. package/src/cli/wizard/plugin-selection.js +0 -297
  214. package/src/cli/wizard/preview-save.js +0 -172
  215. package/src/cli/wizard/project-setup.js +0 -66
  216. package/src/core/pipeline-factory.js +0 -59
  217. package/src/dag/enhanced-dag-engine.js +0 -894
  218. package/src/dx/index.js +0 -40
  219. package/src/dx/integration-templates.js +0 -662
  220. package/src/dx/performance-profiler.js +0 -968
  221. package/src/dx/realtime-debugger.js +0 -755
  222. package/src/dx/visual-pipeline-builder.js +0 -578
  223. package/src/ingest.js +0 -59
  224. package/src/pipeline/pipeline.js +0 -18
  225. package/src/query.js +0 -117
@@ -1,894 +0,0 @@
1
- /**
2
- * @fileoverview Enhanced DAG Engine with Performance Optimizations
3
- * Enterprise-grade DAG execution engine with advanced concurrency, retry policies,
4
- * structured logging, and comprehensive error handling
5
- *
6
- * @author DevilsDev Team
7
- * @since 2.1.8
8
- * @version 2.1.8
9
- */
10
-
11
- const { logger } = require('../utils/structured-logger.js');
12
- const {
13
- defaultPolicy: defaultRetryPolicy,
14
- } = require('../utils/retry-policy.js');
15
-
16
- /**
17
- * @typedef {Object} DAGNodeOptions
18
- * @property {number} [timeout=30000] - Node execution timeout in milliseconds
19
- * @property {number} [retries=0] - Number of retry attempts
20
- * @property {number} [priority=0] - Execution priority (higher executes first)
21
- * @property {boolean} [optional=false] - Whether node failure should be ignored
22
- * @property {Object} [retryPolicy] - Custom retry policy configuration
23
- * @property {string[]} [tags] - Node tags for categorization
24
- * @property {Object} [metadata] - Additional node metadata
25
- */
26
-
27
- /**
28
- * @typedef {Object} ExecutionContext
29
- * @property {Map} results - Map of completed node results
30
- * @property {Map} errors - Map of node execution errors
31
- * @property {Map} fwd - Forward adjacency map for dependencies
32
- * @property {boolean} continueOnError - Whether to continue on non-critical failures
33
- * @property {Set} requiredIds - Set of required node IDs
34
- * @property {any} seed - Initial seed value for source nodes
35
- * @property {boolean} gracefulDegradation - Enable graceful degradation mode
36
- * @property {string[]} requiredNodes - Array of required node IDs
37
- * @property {boolean} retryFailedNodes - Enable retry for failed nodes
38
- * @property {number} maxRetries - Maximum retry attempts per node
39
- */
40
-
41
- /**
42
- * @typedef {Object} ExecutionOptions
43
- * @property {number} [timeout=30000] - Global execution timeout in milliseconds
44
- * @property {number} [concurrency=5] - Maximum concurrent node executions
45
- * @property {boolean} [continueOnError=false] - Continue execution on non-critical failures
46
- * @property {boolean} [gracefulDegradation=false] - Enable graceful degradation mode
47
- * @property {string[]} [requiredNodes] - Array of required node IDs
48
- * @property {boolean} [retryFailedNodes=false] - Enable retry for failed nodes
49
- * @property {number} [maxRetries=3] - Maximum retry attempts per node
50
- * @property {boolean} [enableMetrics=true] - Enable performance metrics collection
51
- * @property {boolean} [enableTracing=false] - Enable execution tracing
52
- */
53
-
54
- /**
55
- * @typedef {Object} NodeMetrics
56
- * @property {number} executionTime - Node execution time in milliseconds
57
- * @property {number} retryCount - Number of retry attempts made
58
- * @property {string} status - Execution status: 'success' | 'failed' | 'skipped'
59
- * @property {Date} startTime - Execution start timestamp
60
- * @property {Date} endTime - Execution end timestamp
61
- * @property {number} memoryUsage - Memory usage during execution
62
- * @property {string[]} dependencies - List of dependency node IDs
63
- */
64
-
65
- /**
66
- * Enhanced DAG Node with performance monitoring and retry capabilities
67
- */
68
- class EnhancedDAGNode {
69
- /**
70
- * Create a new DAG node
71
- * @param {string} id - Unique node identifier
72
- * @param {Function} run - Async function to execute: (input) => output
73
- * @param {DAGNodeOptions} [options={}] - Node configuration options
74
- */
75
- constructor(id, run, options = {}) {
76
- if (!id || typeof id !== 'string') {
77
- throw new Error('Node ID must be a non-empty string');
78
- }
79
- if (!run || typeof run !== 'function') {
80
- throw new Error('Node run function must be provided');
81
- }
82
-
83
- this.id = id;
84
- this.run = run;
85
- this.inputs = [];
86
- this.outputs = [];
87
-
88
- // Configuration options
89
- this.timeout =
90
- options.timeout || parseInt(process.env.RAG_NODE_TIMEOUT) || 30000;
91
- this.retries = options.retries || 0;
92
- this.priority = options.priority || 0;
93
- this.optional = options.optional || false;
94
- this.retryPolicy = options.retryPolicy || null;
95
- this.tags = options.tags || [];
96
- this.metadata = options.metadata || {};
97
-
98
- // Performance tracking
99
- this.metrics = {
100
- executionCount: 0,
101
- totalExecutionTime: 0,
102
- successCount: 0,
103
- failureCount: 0,
104
- retryCount: 0,
105
- lastExecutionTime: null,
106
- avgExecutionTime: 0,
107
- };
108
-
109
- // Create child logger with node context
110
- this.logger = logger.child({
111
- component: 'dag-node',
112
- nodeId: this.id,
113
- priority: this.priority,
114
- optional: this.optional,
115
- });
116
- }
117
-
118
- /**
119
- * Add an output connection to another node
120
- * @param {EnhancedDAGNode} node - Target node to connect to
121
- * @returns {EnhancedDAGNode} This node for chaining
122
- */
123
- addOutput(node) {
124
- if (!node || !(node instanceof EnhancedDAGNode)) {
125
- throw new Error(
126
- 'Invalid output node: must be an EnhancedDAGNode instance',
127
- );
128
- }
129
-
130
- if (!this.outputs.includes(node)) {
131
- this.outputs.push(node);
132
- this.logger.debug('Added output connection', { targetNodeId: node.id });
133
- }
134
- if (!node.inputs.includes(this)) {
135
- node.inputs.push(this);
136
- }
137
- return this;
138
- }
139
-
140
- /**
141
- * Add an input connection from another node
142
- * @param {EnhancedDAGNode} node - Source node to connect from
143
- * @returns {EnhancedDAGNode} This node for chaining
144
- */
145
- addInput(node) {
146
- if (!node || !(node instanceof EnhancedDAGNode)) {
147
- throw new Error(
148
- 'Invalid input node: must be an EnhancedDAGNode instance',
149
- );
150
- }
151
-
152
- if (!this.inputs.includes(node)) {
153
- this.inputs.push(node);
154
- this.logger.debug('Added input connection', { sourceNodeId: node.id });
155
- }
156
- if (!node.outputs.includes(this)) {
157
- node.outputs.push(this);
158
- }
159
- return this;
160
- }
161
-
162
- /**
163
- * Get node dependencies as array of IDs
164
- * @returns {string[]} Array of dependency node IDs
165
- */
166
- getDependencies() {
167
- return this.inputs.map((node) => node.id);
168
- }
169
-
170
- /**
171
- * Check if node has specific tag
172
- * @param {string} tag - Tag to check for
173
- * @returns {boolean} Whether node has the tag
174
- */
175
- hasTag(tag) {
176
- return this.tags.includes(tag);
177
- }
178
-
179
- /**
180
- * Update node metrics after execution
181
- * @param {number} executionTime - Execution time in milliseconds
182
- * @param {boolean} success - Whether execution was successful
183
- * @param {number} retryCount - Number of retries made
184
- */
185
- updateMetrics(executionTime, success, retryCount = 0) {
186
- this.metrics.executionCount++;
187
- this.metrics.totalExecutionTime += executionTime;
188
- this.metrics.retryCount += retryCount;
189
- this.metrics.lastExecutionTime = executionTime;
190
-
191
- if (success) {
192
- this.metrics.successCount++;
193
- } else {
194
- this.metrics.failureCount++;
195
- }
196
-
197
- this.metrics.avgExecutionTime =
198
- this.metrics.totalExecutionTime / this.metrics.executionCount;
199
- }
200
-
201
- /**
202
- * Get node performance metrics
203
- * @returns {NodeMetrics} Node performance metrics
204
- */
205
- getMetrics() {
206
- return {
207
- ...this.metrics,
208
- dependencies: this.getDependencies(),
209
- successRate:
210
- this.metrics.executionCount > 0
211
- ? this.metrics.successCount / this.metrics.executionCount
212
- : 0,
213
- };
214
- }
215
-
216
- /**
217
- * Reset node metrics
218
- */
219
- resetMetrics() {
220
- this.metrics = {
221
- executionCount: 0,
222
- totalExecutionTime: 0,
223
- successCount: 0,
224
- failureCount: 0,
225
- retryCount: 0,
226
- lastExecutionTime: null,
227
- avgExecutionTime: 0,
228
- };
229
- }
230
- }
231
-
232
- /**
233
- * Enhanced DAG Engine with advanced performance and reliability features
234
- */
235
- class EnhancedDAGEngine {
236
- /**
237
- * Create a new enhanced DAG engine
238
- * @param {Object} [options={}] - Engine configuration options
239
- */
240
- constructor(options = {}) {
241
- this.nodes = new Map();
242
- this.executionHistory = [];
243
- this.enableMetrics = options.enableMetrics !== false;
244
- this.enableTracing = options.enableTracing || false;
245
-
246
- // Default execution options
247
- this.defaultOptions = {
248
- timeout: 30000,
249
- concurrency: parseInt(process.env.RAG_MAX_CONCURRENCY) || 5,
250
- continueOnError: false,
251
- gracefulDegradation: false,
252
- retryFailedNodes: false,
253
- maxRetries: 3,
254
- enableMetrics: this.enableMetrics,
255
- enableTracing: this.enableTracing,
256
- };
257
-
258
- // Performance tracking
259
- this.engineMetrics = {
260
- totalExecutions: 0,
261
- successfulExecutions: 0,
262
- failedExecutions: 0,
263
- avgExecutionTime: 0,
264
- totalExecutionTime: 0,
265
- };
266
-
267
- // Create engine logger
268
- this.logger = logger.child({
269
- component: 'enhanced-dag-engine',
270
- version: '2.1.8',
271
- });
272
-
273
- this.logger.info('Enhanced DAG Engine initialized', {
274
- defaultConcurrency: this.defaultOptions.concurrency,
275
- metricsEnabled: this.enableMetrics,
276
- tracingEnabled: this.enableTracing,
277
- });
278
- }
279
-
280
- /**
281
- * Add a node to the DAG
282
- * @param {string} id - Unique node identifier
283
- * @param {Function} run - Async function to execute
284
- * @param {DAGNodeOptions} [options={}] - Node configuration
285
- * @returns {EnhancedDAGNode} The created node
286
- */
287
- addNode(id, run, options = {}) {
288
- if (this.nodes.has(id)) {
289
- throw new Error(`Node with ID '${id}' already exists`);
290
- }
291
-
292
- const node = new EnhancedDAGNode(id, run, options);
293
- this.nodes.set(id, node);
294
-
295
- this.logger.debug('Node added to DAG', {
296
- nodeId: id,
297
- priority: node.priority,
298
- optional: node.optional,
299
- timeout: node.timeout,
300
- });
301
-
302
- return node;
303
- }
304
-
305
- /**
306
- * Get a node by ID
307
- * @param {string} id - Node identifier
308
- * @returns {EnhancedDAGNode|undefined} The node or undefined if not found
309
- */
310
- getNode(id) {
311
- return this.nodes.get(id);
312
- }
313
-
314
- /**
315
- * Connect two nodes
316
- * @param {string} fromId - Source node ID
317
- * @param {string} toId - Target node ID
318
- * @returns {EnhancedDAGEngine} This engine for chaining
319
- */
320
- connect(fromId, toId) {
321
- const fromNode = this.nodes.get(fromId);
322
- const toNode = this.nodes.get(toId);
323
-
324
- if (!fromNode) {
325
- throw new Error(`Source node '${fromId}' not found`);
326
- }
327
- if (!toNode) {
328
- throw new Error(`Target node '${toId}' not found`);
329
- }
330
-
331
- fromNode.addOutput(toNode);
332
-
333
- this.logger.debug('Nodes connected', {
334
- fromNodeId: fromId,
335
- toNodeId: toId,
336
- });
337
-
338
- return this;
339
- }
340
-
341
- /**
342
- * Validate DAG for cycles and structural integrity
343
- * @throws {Error} If DAG is invalid
344
- */
345
- validate() {
346
- this.logger.debug('Validating DAG structure');
347
-
348
- // Check for cycles using DFS
349
- const visited = new Set();
350
- const recursionStack = new Set();
351
-
352
- const hasCycle = (nodeId) => {
353
- if (recursionStack.has(nodeId)) {
354
- return true;
355
- }
356
- if (visited.has(nodeId)) {
357
- return false;
358
- }
359
-
360
- visited.add(nodeId);
361
- recursionStack.add(nodeId);
362
-
363
- const node = this.nodes.get(nodeId);
364
- for (const output of node.outputs) {
365
- if (hasCycle(output.id)) {
366
- return true;
367
- }
368
- }
369
-
370
- recursionStack.delete(nodeId);
371
- return false;
372
- };
373
-
374
- for (const nodeId of this.nodes.keys()) {
375
- if (hasCycle(nodeId)) {
376
- throw new Error(`Cycle detected in DAG involving node: ${nodeId}`);
377
- }
378
- }
379
-
380
- // Validate node integrity
381
- for (const [nodeId, node] of this.nodes) {
382
- // Check that all input/output connections are bidirectional
383
- for (const input of node.inputs) {
384
- if (!input.outputs.includes(node)) {
385
- throw new Error(
386
- `Broken bidirectional connection: ${input.id} -> ${nodeId}`,
387
- );
388
- }
389
- }
390
- for (const output of node.outputs) {
391
- if (!output.inputs.includes(node)) {
392
- throw new Error(
393
- `Broken bidirectional connection: ${nodeId} -> ${output.id}`,
394
- );
395
- }
396
- }
397
- }
398
-
399
- this.logger.info('DAG validation completed successfully', {
400
- nodeCount: this.nodes.size,
401
- });
402
- }
403
-
404
- /**
405
- * Get topological order of nodes for execution
406
- * @returns {EnhancedDAGNode[]} Nodes in execution order
407
- */
408
- getTopologicalOrder() {
409
- const visited = new Set();
410
- const order = [];
411
-
412
- const visit = (node) => {
413
- if (visited.has(node.id)) {
414
- return;
415
- }
416
-
417
- visited.add(node.id);
418
-
419
- // Visit dependencies first (sorted by priority)
420
- const sortedInputs = [...node.inputs].sort(
421
- (a, b) => b.priority - a.priority,
422
- );
423
- for (const input of sortedInputs) {
424
- visit(input);
425
- }
426
-
427
- order.push(node);
428
- };
429
-
430
- // Start with nodes that have no inputs, sorted by priority
431
- const sourceNodes = Array.from(this.nodes.values())
432
- .filter((node) => node.inputs.length === 0)
433
- .sort((a, b) => b.priority - a.priority);
434
-
435
- for (const node of sourceNodes) {
436
- visit(node);
437
- }
438
-
439
- // Ensure all nodes are included
440
- for (const node of this.nodes.values()) {
441
- visit(node);
442
- }
443
-
444
- this.logger.debug('Topological order computed', {
445
- nodeCount: order.length,
446
- sourceNodes: sourceNodes.map((n) => n.id),
447
- });
448
-
449
- return order;
450
- }
451
-
452
- /**
453
- * Execute a single node with comprehensive error handling and retry logic
454
- * @param {EnhancedDAGNode} node - Node to execute
455
- * @param {ExecutionContext} context - Execution context
456
- * @returns {Promise<any>} Node execution result
457
- */
458
- async executeNode(node, context) {
459
- const { results, errors, seed, maxRetries } = context;
460
- const startTime = Date.now();
461
- let retryCount = 0;
462
-
463
- const correlationId = this.logger.getContext()?.correlationId;
464
-
465
- node.logger.debug('Starting node execution', {
466
- correlationId,
467
- dependencies: node.getDependencies(),
468
- hasResults: node.inputs.map((dep) => results.has(dep.id)),
469
- });
470
-
471
- try {
472
- // Prepare input data from dependencies
473
- let input = seed;
474
- if (node.inputs.length > 0) {
475
- input = {};
476
- for (const dep of node.inputs) {
477
- if (results.has(dep.id)) {
478
- input[dep.id] = results.get(dep.id);
479
- }
480
- }
481
- }
482
-
483
- // Create execution function with timeout
484
- const executeWithTimeout = async () => {
485
- const timeoutPromise = new Promise((_, reject) => {
486
- setTimeout(() => {
487
- reject(
488
- new Error(
489
- `Node ${node.id} execution timeout after ${node.timeout}ms`,
490
- ),
491
- );
492
- }, node.timeout);
493
- });
494
-
495
- return Promise.race([node.run(input), timeoutPromise]);
496
- };
497
-
498
- // Use retry policy if configured
499
- let result;
500
- if (node.retryPolicy || (node.retries > 0 && maxRetries > 0)) {
501
- const retryPolicy = node.retryPolicy || defaultRetryPolicy;
502
-
503
- result = await retryPolicy.execute(executeWithTimeout, {
504
- nodeId: node.id,
505
- correlationId,
506
- });
507
- retryCount = retryPolicy.getMetrics().retryBudget.currentRetries;
508
- } else {
509
- result = await executeWithTimeout();
510
- }
511
-
512
- const executionTime = Date.now() - startTime;
513
-
514
- // Update metrics
515
- if (this.enableMetrics) {
516
- node.updateMetrics(executionTime, true, retryCount);
517
- }
518
-
519
- node.logger.info('Node execution completed successfully', {
520
- correlationId,
521
- executionTime,
522
- retryCount,
523
- resultType: typeof result,
524
- });
525
-
526
- return result;
527
- } catch (error) {
528
- const executionTime = Date.now() - startTime;
529
-
530
- // Update metrics
531
- if (this.enableMetrics) {
532
- node.updateMetrics(executionTime, false, retryCount);
533
- }
534
-
535
- // Handle optional node failures
536
- if (node.optional && context.gracefulDegradation) {
537
- node.logger.warn(
538
- 'Optional node failed, continuing with graceful degradation',
539
- {
540
- correlationId,
541
- error: error.message,
542
- executionTime,
543
- retryCount,
544
- },
545
- );
546
- return Symbol('failed-optional-node');
547
- }
548
-
549
- // Store error for required nodes
550
- errors.set(node.id, error);
551
-
552
- node.logger.error('Node execution failed', {
553
- correlationId,
554
- error: error.message,
555
- errorCode: error.code,
556
- executionTime,
557
- retryCount,
558
- stack: error.stack,
559
- });
560
-
561
- // Re-throw for required nodes
562
- throw error;
563
- }
564
- }
565
-
566
- /**
567
- * Execute the DAG with advanced concurrency and error handling
568
- * @param {any} [seed] - Initial seed value for source nodes
569
- * @param {ExecutionOptions} [options={}] - Execution options
570
- * @returns {Promise<Object>} Execution results
571
- */
572
- async execute(seed, options = {}) {
573
- const executionId = require('crypto').randomUUID();
574
- const startTime = Date.now();
575
-
576
- // Merge options with defaults
577
- const config = { ...this.defaultOptions, ...options };
578
-
579
- return this.logger.withCorrelation(executionId, async () => {
580
- this.logger.info('Starting DAG execution', {
581
- executionId,
582
- seedType: typeof seed,
583
- nodeCount: this.nodes.size,
584
- concurrency: config.concurrency,
585
- timeout: config.timeout,
586
- });
587
-
588
- try {
589
- // Validate DAG before execution
590
- this.validate();
591
-
592
- // Initialize execution state and context
593
- const results = new Map();
594
- const errors = new Map();
595
- const order = this.getTopologicalOrder();
596
- const context = this._buildExecutionContext(
597
- results,
598
- errors,
599
- seed,
600
- config,
601
- );
602
-
603
- // Schedule and run all nodes with concurrency control
604
- await this._scheduleNodes(order, context, config);
605
-
606
- const executionTime = Date.now() - startTime;
607
-
608
- // Update engine metrics
609
- this._recordExecutionMetrics(executionTime, errors.size === 0);
610
-
611
- // Build and store execution summary
612
- const summary = this._buildExecutionSummary(
613
- executionId,
614
- executionTime,
615
- results,
616
- errors,
617
- );
618
- this._storeExecutionHistory(summary, results);
619
-
620
- this.logger.info('DAG execution completed', summary);
621
-
622
- return {
623
- results: Object.fromEntries(results),
624
- errors: Object.fromEntries(errors),
625
- summary,
626
- metrics: this.enableMetrics ? this.getMetrics() : null,
627
- };
628
- } catch (error) {
629
- const executionTime = Date.now() - startTime;
630
- this._recordExecutionFailure(executionId, error, executionTime);
631
- throw error;
632
- }
633
- });
634
- }
635
-
636
- /**
637
- * Build the execution context passed to each node during DAG execution.
638
- * @param {Map} results - Map to store completed node results
639
- * @param {Map} errors - Map to store node execution errors
640
- * @param {any} seed - Initial seed value for source nodes
641
- * @param {Object} config - Merged execution configuration
642
- * @returns {ExecutionContext} Execution context object
643
- * @private
644
- */
645
- _buildExecutionContext(results, errors, seed, config) {
646
- return {
647
- results,
648
- errors,
649
- seed,
650
- continueOnError: config.continueOnError,
651
- gracefulDegradation: config.gracefulDegradation,
652
- retryFailedNodes: config.retryFailedNodes,
653
- maxRetries: config.maxRetries,
654
- };
655
- }
656
-
657
- /**
658
- * Schedule and execute all nodes in topological order with concurrency control.
659
- * Each node waits for its dependencies before executing.
660
- * @param {EnhancedDAGNode[]} order - Nodes in topological execution order
661
- * @param {ExecutionContext} context - Execution context with results and errors maps
662
- * @param {Object} config - Execution configuration including concurrency limit
663
- * @returns {Promise<void>} Resolves when all nodes have settled
664
- * @private
665
- */
666
- async _scheduleNodes(order, context, config) {
667
- const { results } = context;
668
- const semaphore = new Semaphore(config.concurrency);
669
- const nodePromises = new Map();
670
-
671
- for (const node of order) {
672
- const promise = semaphore.acquire().then(async (release) => {
673
- try {
674
- // Wait for dependencies to complete
675
- const depPromises = node.inputs
676
- .map((dep) => nodePromises.get(dep.id))
677
- .filter(Boolean);
678
-
679
- await Promise.all(depPromises);
680
-
681
- // Execute the node
682
- const result = await this.executeNode(node, context);
683
-
684
- if (
685
- typeof result !== 'symbol' ||
686
- result.description !== 'failed-optional-node'
687
- ) {
688
- results.set(node.id, result);
689
- }
690
-
691
- return result;
692
- } finally {
693
- release();
694
- }
695
- });
696
-
697
- nodePromises.set(node.id, promise);
698
- }
699
-
700
- await Promise.allSettled(Array.from(nodePromises.values()));
701
- }
702
-
703
- /**
704
- * Record engine-level metrics after a completed execution.
705
- * @param {number} executionTime - Total execution time in milliseconds
706
- * @param {boolean} success - Whether the execution completed without errors
707
- * @private
708
- */
709
- _recordExecutionMetrics(executionTime, success) {
710
- if (!this.enableMetrics) {
711
- return;
712
- }
713
-
714
- this.engineMetrics.totalExecutions++;
715
- this.engineMetrics.totalExecutionTime += executionTime;
716
- this.engineMetrics.avgExecutionTime =
717
- this.engineMetrics.totalExecutionTime /
718
- this.engineMetrics.totalExecutions;
719
-
720
- if (success) {
721
- this.engineMetrics.successfulExecutions++;
722
- } else {
723
- this.engineMetrics.failedExecutions++;
724
- }
725
- }
726
-
727
- /**
728
- * Build a summary object describing the outcome of a DAG execution.
729
- * @param {string} executionId - Unique execution identifier
730
- * @param {number} executionTime - Total execution time in milliseconds
731
- * @param {Map} results - Map of successful node results
732
- * @param {Map} errors - Map of node execution errors
733
- * @returns {Object} Execution summary
734
- * @private
735
- */
736
- _buildExecutionSummary(executionId, executionTime, results, errors) {
737
- return {
738
- executionId,
739
- success: errors.size === 0,
740
- executionTime,
741
- nodesExecuted: results.size,
742
- totalNodes: this.nodes.size,
743
- errors: Array.from(errors.entries()).map(([id, error]) => ({
744
- nodeId: id,
745
- message: error.message,
746
- code: error.code,
747
- })),
748
- };
749
- }
750
-
751
- /**
752
- * Store an execution summary in the tracing history buffer.
753
- * Keeps only the most recent 100 entries.
754
- * @param {Object} summary - Execution summary to store
755
- * @param {Map} results - Map of node results to include in history
756
- * @private
757
- */
758
- _storeExecutionHistory(summary, results) {
759
- if (!this.enableTracing) {
760
- return;
761
- }
762
-
763
- this.executionHistory.push({
764
- ...summary,
765
- timestamp: new Date().toISOString(),
766
- results: Object.fromEntries(results),
767
- });
768
-
769
- if (this.executionHistory.length > 100) {
770
- this.executionHistory.shift();
771
- }
772
- }
773
-
774
- /**
775
- * Record metrics and log details when a DAG execution fails with an exception.
776
- * @param {string} executionId - Unique execution identifier
777
- * @param {Error} error - The error that caused the failure
778
- * @param {number} executionTime - Time elapsed before the failure
779
- * @private
780
- */
781
- _recordExecutionFailure(executionId, error, executionTime) {
782
- if (this.enableMetrics) {
783
- this.engineMetrics.totalExecutions++;
784
- this.engineMetrics.failedExecutions++;
785
- }
786
-
787
- this.logger.error('DAG execution failed', {
788
- executionId,
789
- error: error.message,
790
- executionTime,
791
- stack: error.stack,
792
- });
793
- }
794
-
795
- /**
796
- * Get comprehensive performance metrics
797
- * @returns {Object} Performance metrics
798
- */
799
- getMetrics() {
800
- const nodeMetrics = {};
801
- for (const [id, node] of this.nodes) {
802
- nodeMetrics[id] = node.getMetrics();
803
- }
804
-
805
- return {
806
- engine: this.engineMetrics,
807
- nodes: nodeMetrics,
808
- summary: {
809
- totalNodes: this.nodes.size,
810
- avgSuccessRate:
811
- Object.values(nodeMetrics).reduce(
812
- (sum, metrics) => sum + metrics.successRate,
813
- 0,
814
- ) / this.nodes.size,
815
- },
816
- };
817
- }
818
-
819
- /**
820
- * Reset all metrics
821
- */
822
- resetMetrics() {
823
- this.engineMetrics = {
824
- totalExecutions: 0,
825
- successfulExecutions: 0,
826
- failedExecutions: 0,
827
- avgExecutionTime: 0,
828
- totalExecutionTime: 0,
829
- };
830
-
831
- for (const node of this.nodes.values()) {
832
- node.resetMetrics();
833
- }
834
-
835
- this.executionHistory = [];
836
- }
837
-
838
- /**
839
- * Get execution history (if tracing is enabled)
840
- * @returns {Array} Execution history
841
- */
842
- getExecutionHistory() {
843
- return [...this.executionHistory];
844
- }
845
- }
846
-
847
- /**
848
- * Semaphore for concurrency control
849
- */
850
- class Semaphore {
851
- constructor(maxConcurrency) {
852
- this.maxConcurrency = maxConcurrency;
853
- this.currentCount = 0;
854
- this.waitQueue = [];
855
- }
856
-
857
- async acquire() {
858
- return new Promise((resolve) => {
859
- if (this.currentCount < this.maxConcurrency) {
860
- this.currentCount++;
861
- resolve(() => this.release());
862
- } else {
863
- this.waitQueue.push(() => {
864
- this.currentCount++;
865
- resolve(() => this.release());
866
- });
867
- }
868
- });
869
- }
870
-
871
- release() {
872
- this.currentCount--;
873
- if (this.waitQueue.length > 0) {
874
- const next = this.waitQueue.shift();
875
- next();
876
- }
877
- }
878
- }
879
-
880
- /**
881
- * Create enhanced DAG engine instance
882
- * @param {Object} [options={}] - Engine configuration
883
- * @returns {EnhancedDAGEngine} DAG engine instance
884
- */
885
- function createEnhancedDAG(options = {}) {
886
- return new EnhancedDAGEngine(options);
887
- }
888
-
889
- module.exports = {
890
- EnhancedDAGEngine,
891
- EnhancedDAGNode,
892
- Semaphore,
893
- createEnhancedDAG,
894
- };