@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ragrc.schema.json +83 -192
- package/CHANGELOG.md +345 -0
- package/README.md +418 -117
- package/SECURITY.md +5 -5
- package/dist/experimental.cjs +2 -0
- package/dist/experimental.d.ts +1 -0
- package/dist/experimental.mjs +5 -0
- package/dist/index.cjs +4992 -1242
- package/dist/index.d.ts +80 -2
- package/dist/index.mjs +1 -1
- package/dist/types/agentic/agentic-pipeline.d.ts +103 -0
- package/dist/types/agentic/iterative-retriever.d.ts +85 -0
- package/dist/types/agentic/query-planner.d.ts +145 -0
- package/dist/types/agentic/self-critique.d.ts +45 -0
- package/dist/types/ai/adaptive-retrieval.d.ts +44 -0
- package/dist/types/ai/federation/federated-learning-coordinator.d.ts +62 -0
- package/dist/types/ai/multimodal-processing.d.ts +32 -0
- package/dist/types/ai/multimodal.d.ts +39 -0
- package/dist/types/ai/retrieval-engine.d.ts +37 -0
- package/dist/types/ai/training/model-training-orchestrator.d.ts +31 -0
- package/dist/types/chunking/chunking-engine.d.ts +70 -0
- package/dist/types/chunking/strategies/fixed-size.d.ts +17 -0
- package/dist/types/chunking/strategies/recursive.d.ts +20 -0
- package/dist/types/chunking/strategies/semantic.d.ts +19 -0
- package/dist/types/chunking/strategies/sentence.d.ts +13 -0
- package/dist/types/chunking/strategies/structure-aware.d.ts +19 -0
- package/dist/types/citation/citation-tracker.d.ts +64 -0
- package/dist/types/citation/hallucination-detector.d.ts +40 -0
- package/dist/types/citation/source-mapper.d.ts +37 -0
- package/dist/types/connectors/anthropic-connector.d.ts +92 -0
- package/dist/types/connectors/base-connector.d.ts +87 -0
- package/dist/types/connectors/cohere-connector.d.ts +89 -0
- package/dist/types/connectors/connector-utils.d.ts +110 -0
- package/dist/types/connectors/local-embedder.d.ts +86 -0
- package/dist/types/connectors/memory-retriever.d.ts +91 -0
- package/dist/types/connectors/ollama-connector.d.ts +110 -0
- package/dist/types/connectors/openai-connector.d.ts +114 -0
- package/dist/types/connectors/rag-prompt.d.ts +40 -0
- package/dist/types/cost/cost-calculator.d.ts +152 -0
- package/dist/types/cost/cost-tracker.d.ts +126 -0
- package/dist/types/cost/token-budget.d.ts +110 -0
- package/dist/types/dag/core/error-context.d.ts +239 -0
- package/dist/types/dag/core/execution-scheduler.d.ts +191 -0
- package/dist/types/dag/core/topology-validation.d.ts +226 -0
- package/dist/types/dag/dag-engine.d.ts +156 -0
- package/dist/types/dag/dag-node.d.ts +37 -0
- package/dist/types/dashboard/dashboard-generator.d.ts +127 -0
- package/dist/types/dashboard/metrics-aggregator.d.ts +218 -0
- package/dist/types/debug/execution-tracer.d.ts +57 -0
- package/dist/types/debug/trace-inspector.d.ts +31 -0
- package/dist/types/ecosystem/plugin-certification.d.ts +169 -0
- package/dist/types/enterprise/data-governance.d.ts +203 -0
- package/dist/types/enterprise/multi-tenancy.d.ts +137 -0
- package/dist/types/enterprise/sso-integration.d.ts +420 -0
- package/dist/types/evaluate/context-metrics.d.ts +55 -0
- package/dist/types/evaluate/faithfulness.d.ts +41 -0
- package/dist/types/evaluate/groundedness.d.ts +60 -0
- package/dist/types/evaluate/pipeline-evaluator.d.ts +88 -0
- package/dist/types/evaluate/relevance.d.ts +21 -0
- package/dist/types/evaluate/scoring.d.ts +123 -0
- package/dist/types/experimental.d.ts +15 -0
- package/dist/types/graph/entity-extractor.d.ts +84 -0
- package/dist/types/graph/graph-index.d.ts +49 -0
- package/dist/types/graph/graph-retriever.d.ts +81 -0
- package/dist/types/graph/knowledge-graph.d.ts +129 -0
- package/dist/types/guardrails/guardrails-pipeline.d.ts +63 -0
- package/dist/types/guardrails/injection-detector.d.ts +47 -0
- package/dist/types/guardrails/post-generation-guard.d.ts +61 -0
- package/dist/types/guardrails/pre-retrieval-guard.d.ts +64 -0
- package/dist/types/guardrails/retrieval-guard.d.ts +57 -0
- package/dist/types/mcp/mcp-server.d.ts +85 -0
- package/dist/types/mcp/mcp-tool-builder.d.ts +55 -0
- package/dist/types/reranker/cascade-reranker.d.ts +40 -0
- package/dist/types/reranker/embedding-reranker.d.ts +77 -0
- package/dist/types/reranker/scoring-reranker.d.ts +41 -0
- package/dist/types/retrieval/bm25-search.d.ts +43 -0
- package/dist/types/retrieval/file-vector-store.d.ts +87 -0
- package/dist/types/retrieval/hybrid-retriever.d.ts +35 -0
- package/dist/types/retrieval/rank-fusion.d.ts +10 -0
- package/dist/types/security/jwt-config.d.ts +80 -0
- package/dist/types/security/jwt-validator.d.ts +195 -0
- package/dist/types/security/plugin-sandbox.d.ts +158 -0
- package/dist/types/security/plugin-signature-verifier.d.ts +139 -0
- package/dist/types/security/validator.d.ts +5 -0
- package/dist/types/streaming/embedding-stream.d.ts +65 -0
- package/dist/types/streaming/sse-adapter.d.ts +59 -0
- package/dist/types/streaming/stream-router.d.ts +88 -0
- package/dist/types/streaming/streaming-embedder.d.ts +99 -0
- package/dist/types/streaming/websocket-adapter.d.ts +71 -0
- package/dist/types/templates/scaffold.d.ts +55 -0
- package/dist/types/templates/template-registry.d.ts +63 -0
- package/dist/types/utils/embed-query.d.ts +20 -0
- package/dist/types/utils/logger.d.ts +50 -0
- package/dist/types/utils/rate-limiter.d.ts +66 -0
- package/dist/types/utils/retry.d.ts +27 -0
- package/dist/types/utils/run-abort.d.ts +34 -0
- package/dist/types/utils/secure-logging.d.ts +51 -0
- package/dist/types/utils/structured-logger.d.ts +126 -0
- package/package.json +53 -29
- package/src/agentic/agentic-pipeline.js +75 -7
- package/src/agentic/iterative-retriever.js +47 -10
- package/src/agentic/query-planner.js +329 -78
- package/src/citation/citation-tracker.js +6 -4
- package/src/citation/hallucination-detector.js +2 -2
- package/src/citation/source-mapper.js +15 -28
- package/src/cli/commands/ai.js +101 -0
- package/src/cli/commands/plugin-hub/event-handlers.js +6 -0
- package/src/cli/commands/plugin-hub/hub-commands.js +0 -74
- package/src/cli/commands/plugin-hub/local-commands.js +143 -0
- package/src/cli/commands/plugin-hub.js +96 -36
- package/src/cli/diagnostics/config-checker.js +76 -63
- package/src/cli/diagnostics/environment-checker.js +62 -31
- package/src/cli/diagnostics/plugin-checker.js +36 -89
- package/src/cli/doctor-command.js +39 -5
- package/src/cli/enhanced-cli-commands.js +72 -217
- package/src/cli/handlers/ai-handler.js +364 -0
- package/src/cli/handlers/diagnostics-handler.js +78 -31
- package/src/cli/handlers/ingest-handler.js +28 -22
- package/src/cli/handlers/init-handler.js +148 -25
- package/src/cli/handlers/query-handler.js +41 -26
- package/src/cli/handlers/run-handler.js +16 -167
- package/src/cli/handlers/serve-handler.js +60 -0
- package/src/cli/services/bench-stats.js +54 -0
- package/src/cli/services/benchmark.js +161 -0
- package/src/cli/services/feedback.js +155 -0
- package/src/cli/services/rag-service.js +322 -0
- package/src/cli/shared/cli-utils.js +24 -7
- package/src/cli/shared/config-context.js +21 -0
- package/src/cli/wizard/config-builder.js +81 -0
- package/src/cli/wizard/init-wizard.js +617 -0
- package/src/config/component-factory.js +125 -0
- package/src/config/load-config.js +29 -10
- package/src/config/load-plugin-config.js +46 -2
- package/src/connectors/anthropic-connector.js +93 -55
- package/src/connectors/base-connector.js +74 -19
- package/src/connectors/cohere-connector.js +88 -32
- package/src/connectors/connector-utils.js +302 -0
- package/src/connectors/local-embedder.js +43 -5
- package/src/connectors/memory-retriever.js +108 -15
- package/src/connectors/ollama-connector.js +83 -49
- package/src/connectors/openai-connector.js +108 -44
- package/src/connectors/rag-prompt.js +88 -0
- package/src/core/create-pipeline.js +209 -60
- package/src/core/plugin-registry.js +87 -11
- package/src/cost/cost-calculator.js +205 -32
- package/src/dev/dev-server.js +166 -11
- package/src/dev/hot-reload.js +40 -1
- package/src/ecosystem/plugin-checks.js +181 -0
- package/src/ecosystem/plugin-hub.js +50 -125
- package/src/ecosystem/plugin-installer.js +298 -0
- package/src/ecosystem/plugin-probe.js +127 -0
- package/src/ecosystem/plugin-scanner.js +185 -0
- package/src/ecosystem/tar-extract.js +114 -0
- package/src/enterprise/audit-logging.js +158 -29
- package/src/enterprise/sso-integration.js +715 -228
- package/src/evaluate/context-metrics.js +28 -16
- package/src/evaluate/evaluator.js +3 -3
- package/src/evaluate/faithfulness.js +18 -14
- package/src/evaluate/groundedness.js +43 -12
- package/src/evaluate/pipeline-evaluator.js +35 -8
- package/src/evaluate/scoring.js +251 -3
- package/src/experimental.js +71 -0
- package/src/guardrails/guardrails-pipeline.js +69 -7
- package/src/guardrails/injection-detector.js +283 -0
- package/src/guardrails/post-generation-guard.js +19 -0
- package/src/guardrails/pre-retrieval-guard.js +36 -36
- package/src/guardrails/retrieval-guard.js +58 -8
- package/src/index.js +36 -1
- package/src/loader/document-reader.js +91 -0
- package/src/reranker/cascade-reranker.js +3 -0
- package/src/reranker/embedding-reranker.js +136 -21
- package/src/reranker/llm-reranker.js +79 -12
- package/src/reranker/scoring-reranker.js +3 -0
- package/src/retrieval/bm25-search.js +12 -6
- package/src/retrieval/file-vector-store.js +212 -0
- package/src/retrieval/hybrid-retriever.js +22 -19
- package/src/security/jwt-config.js +13 -2
- package/src/security/jwt-validator.js +134 -30
- package/src/security/plugin-sandbox.js +18 -1
- package/src/security/plugin-signature-verifier.js +245 -70
- package/src/utils/deprecate.js +69 -0
- package/src/utils/embed-query.js +33 -0
- package/src/utils/input-sanitizer.js +19 -4
- package/src/utils/logger.js +34 -4
- package/src/utils/run-abort.js +102 -0
- package/src/ai/federated-learning.js +0 -211
- package/src/ai/federation.js +0 -71
- package/src/ai/index.js +0 -22
- package/src/ai/model-trainer.js +0 -217
- package/src/ai/model-training.js +0 -202
- package/src/ai/multimodal/multi-modal-processor.js +0 -345
- package/src/ai/orchestrator.js +0 -65
- package/src/ai/retrieval/adaptive-retrieval-engine.js +0 -398
- package/src/cli/commands/ai-ml/adaptive-commands.js +0 -178
- package/src/cli/commands/ai-ml/federated-commands.js +0 -207
- package/src/cli/commands/ai-ml/index.js +0 -17
- package/src/cli/commands/ai-ml/multimodal-commands.js +0 -187
- package/src/cli/commands/ai-ml/shared.js +0 -51
- package/src/cli/commands/ai-ml/training-commands.js +0 -228
- package/src/cli/commands/ai-ml.js +0 -386
- package/src/cli/commands/docs.js +0 -228
- package/src/cli/commands/dx.js +0 -538
- package/src/cli/commands/wizard.js +0 -627
- package/src/cli/input-sanitization-middleware.js +0 -356
- package/src/cli/interactive-wizard.js +0 -373
- package/src/cli/templates/development.template.json +0 -64
- package/src/cli/templates/minimal.template.json +0 -33
- package/src/cli/templates/production.template.json +0 -104
- package/src/cli/templates/testing.template.json +0 -45
- package/src/cli/wizard/configuration.js +0 -148
- package/src/cli/wizard/observability.js +0 -79
- package/src/cli/wizard/performance.js +0 -76
- package/src/cli/wizard/plugin-selection.js +0 -297
- package/src/cli/wizard/preview-save.js +0 -172
- package/src/cli/wizard/project-setup.js +0 -66
- package/src/core/pipeline-factory.js +0 -59
- package/src/dag/enhanced-dag-engine.js +0 -894
- package/src/dx/index.js +0 -40
- package/src/dx/integration-templates.js +0 -662
- package/src/dx/performance-profiler.js +0 -968
- package/src/dx/realtime-debugger.js +0 -755
- package/src/dx/visual-pipeline-builder.js +0 -578
- package/src/ingest.js +0 -59
- package/src/pipeline/pipeline.js +0 -18
- package/src/query.js +0 -117
|
@@ -1,894 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview Enhanced DAG Engine with Performance Optimizations
|
|
3
|
-
* Enterprise-grade DAG execution engine with advanced concurrency, retry policies,
|
|
4
|
-
* structured logging, and comprehensive error handling
|
|
5
|
-
*
|
|
6
|
-
* @author DevilsDev Team
|
|
7
|
-
* @since 2.1.8
|
|
8
|
-
* @version 2.1.8
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
const { logger } = require('../utils/structured-logger.js');
|
|
12
|
-
const {
|
|
13
|
-
defaultPolicy: defaultRetryPolicy,
|
|
14
|
-
} = require('../utils/retry-policy.js');
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* @typedef {Object} DAGNodeOptions
|
|
18
|
-
* @property {number} [timeout=30000] - Node execution timeout in milliseconds
|
|
19
|
-
* @property {number} [retries=0] - Number of retry attempts
|
|
20
|
-
* @property {number} [priority=0] - Execution priority (higher executes first)
|
|
21
|
-
* @property {boolean} [optional=false] - Whether node failure should be ignored
|
|
22
|
-
* @property {Object} [retryPolicy] - Custom retry policy configuration
|
|
23
|
-
* @property {string[]} [tags] - Node tags for categorization
|
|
24
|
-
* @property {Object} [metadata] - Additional node metadata
|
|
25
|
-
*/
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* @typedef {Object} ExecutionContext
|
|
29
|
-
* @property {Map} results - Map of completed node results
|
|
30
|
-
* @property {Map} errors - Map of node execution errors
|
|
31
|
-
* @property {Map} fwd - Forward adjacency map for dependencies
|
|
32
|
-
* @property {boolean} continueOnError - Whether to continue on non-critical failures
|
|
33
|
-
* @property {Set} requiredIds - Set of required node IDs
|
|
34
|
-
* @property {any} seed - Initial seed value for source nodes
|
|
35
|
-
* @property {boolean} gracefulDegradation - Enable graceful degradation mode
|
|
36
|
-
* @property {string[]} requiredNodes - Array of required node IDs
|
|
37
|
-
* @property {boolean} retryFailedNodes - Enable retry for failed nodes
|
|
38
|
-
* @property {number} maxRetries - Maximum retry attempts per node
|
|
39
|
-
*/
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* @typedef {Object} ExecutionOptions
|
|
43
|
-
* @property {number} [timeout=30000] - Global execution timeout in milliseconds
|
|
44
|
-
* @property {number} [concurrency=5] - Maximum concurrent node executions
|
|
45
|
-
* @property {boolean} [continueOnError=false] - Continue execution on non-critical failures
|
|
46
|
-
* @property {boolean} [gracefulDegradation=false] - Enable graceful degradation mode
|
|
47
|
-
* @property {string[]} [requiredNodes] - Array of required node IDs
|
|
48
|
-
* @property {boolean} [retryFailedNodes=false] - Enable retry for failed nodes
|
|
49
|
-
* @property {number} [maxRetries=3] - Maximum retry attempts per node
|
|
50
|
-
* @property {boolean} [enableMetrics=true] - Enable performance metrics collection
|
|
51
|
-
* @property {boolean} [enableTracing=false] - Enable execution tracing
|
|
52
|
-
*/
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* @typedef {Object} NodeMetrics
|
|
56
|
-
* @property {number} executionTime - Node execution time in milliseconds
|
|
57
|
-
* @property {number} retryCount - Number of retry attempts made
|
|
58
|
-
* @property {string} status - Execution status: 'success' | 'failed' | 'skipped'
|
|
59
|
-
* @property {Date} startTime - Execution start timestamp
|
|
60
|
-
* @property {Date} endTime - Execution end timestamp
|
|
61
|
-
* @property {number} memoryUsage - Memory usage during execution
|
|
62
|
-
* @property {string[]} dependencies - List of dependency node IDs
|
|
63
|
-
*/
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Enhanced DAG Node with performance monitoring and retry capabilities
|
|
67
|
-
*/
|
|
68
|
-
class EnhancedDAGNode {
|
|
69
|
-
/**
|
|
70
|
-
* Create a new DAG node
|
|
71
|
-
* @param {string} id - Unique node identifier
|
|
72
|
-
* @param {Function} run - Async function to execute: (input) => output
|
|
73
|
-
* @param {DAGNodeOptions} [options={}] - Node configuration options
|
|
74
|
-
*/
|
|
75
|
-
constructor(id, run, options = {}) {
|
|
76
|
-
if (!id || typeof id !== 'string') {
|
|
77
|
-
throw new Error('Node ID must be a non-empty string');
|
|
78
|
-
}
|
|
79
|
-
if (!run || typeof run !== 'function') {
|
|
80
|
-
throw new Error('Node run function must be provided');
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
this.id = id;
|
|
84
|
-
this.run = run;
|
|
85
|
-
this.inputs = [];
|
|
86
|
-
this.outputs = [];
|
|
87
|
-
|
|
88
|
-
// Configuration options
|
|
89
|
-
this.timeout =
|
|
90
|
-
options.timeout || parseInt(process.env.RAG_NODE_TIMEOUT) || 30000;
|
|
91
|
-
this.retries = options.retries || 0;
|
|
92
|
-
this.priority = options.priority || 0;
|
|
93
|
-
this.optional = options.optional || false;
|
|
94
|
-
this.retryPolicy = options.retryPolicy || null;
|
|
95
|
-
this.tags = options.tags || [];
|
|
96
|
-
this.metadata = options.metadata || {};
|
|
97
|
-
|
|
98
|
-
// Performance tracking
|
|
99
|
-
this.metrics = {
|
|
100
|
-
executionCount: 0,
|
|
101
|
-
totalExecutionTime: 0,
|
|
102
|
-
successCount: 0,
|
|
103
|
-
failureCount: 0,
|
|
104
|
-
retryCount: 0,
|
|
105
|
-
lastExecutionTime: null,
|
|
106
|
-
avgExecutionTime: 0,
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
// Create child logger with node context
|
|
110
|
-
this.logger = logger.child({
|
|
111
|
-
component: 'dag-node',
|
|
112
|
-
nodeId: this.id,
|
|
113
|
-
priority: this.priority,
|
|
114
|
-
optional: this.optional,
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Add an output connection to another node
|
|
120
|
-
* @param {EnhancedDAGNode} node - Target node to connect to
|
|
121
|
-
* @returns {EnhancedDAGNode} This node for chaining
|
|
122
|
-
*/
|
|
123
|
-
addOutput(node) {
|
|
124
|
-
if (!node || !(node instanceof EnhancedDAGNode)) {
|
|
125
|
-
throw new Error(
|
|
126
|
-
'Invalid output node: must be an EnhancedDAGNode instance',
|
|
127
|
-
);
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
if (!this.outputs.includes(node)) {
|
|
131
|
-
this.outputs.push(node);
|
|
132
|
-
this.logger.debug('Added output connection', { targetNodeId: node.id });
|
|
133
|
-
}
|
|
134
|
-
if (!node.inputs.includes(this)) {
|
|
135
|
-
node.inputs.push(this);
|
|
136
|
-
}
|
|
137
|
-
return this;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Add an input connection from another node
|
|
142
|
-
* @param {EnhancedDAGNode} node - Source node to connect from
|
|
143
|
-
* @returns {EnhancedDAGNode} This node for chaining
|
|
144
|
-
*/
|
|
145
|
-
addInput(node) {
|
|
146
|
-
if (!node || !(node instanceof EnhancedDAGNode)) {
|
|
147
|
-
throw new Error(
|
|
148
|
-
'Invalid input node: must be an EnhancedDAGNode instance',
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
if (!this.inputs.includes(node)) {
|
|
153
|
-
this.inputs.push(node);
|
|
154
|
-
this.logger.debug('Added input connection', { sourceNodeId: node.id });
|
|
155
|
-
}
|
|
156
|
-
if (!node.outputs.includes(this)) {
|
|
157
|
-
node.outputs.push(this);
|
|
158
|
-
}
|
|
159
|
-
return this;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* Get node dependencies as array of IDs
|
|
164
|
-
* @returns {string[]} Array of dependency node IDs
|
|
165
|
-
*/
|
|
166
|
-
getDependencies() {
|
|
167
|
-
return this.inputs.map((node) => node.id);
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Check if node has specific tag
|
|
172
|
-
* @param {string} tag - Tag to check for
|
|
173
|
-
* @returns {boolean} Whether node has the tag
|
|
174
|
-
*/
|
|
175
|
-
hasTag(tag) {
|
|
176
|
-
return this.tags.includes(tag);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* Update node metrics after execution
|
|
181
|
-
* @param {number} executionTime - Execution time in milliseconds
|
|
182
|
-
* @param {boolean} success - Whether execution was successful
|
|
183
|
-
* @param {number} retryCount - Number of retries made
|
|
184
|
-
*/
|
|
185
|
-
updateMetrics(executionTime, success, retryCount = 0) {
|
|
186
|
-
this.metrics.executionCount++;
|
|
187
|
-
this.metrics.totalExecutionTime += executionTime;
|
|
188
|
-
this.metrics.retryCount += retryCount;
|
|
189
|
-
this.metrics.lastExecutionTime = executionTime;
|
|
190
|
-
|
|
191
|
-
if (success) {
|
|
192
|
-
this.metrics.successCount++;
|
|
193
|
-
} else {
|
|
194
|
-
this.metrics.failureCount++;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
this.metrics.avgExecutionTime =
|
|
198
|
-
this.metrics.totalExecutionTime / this.metrics.executionCount;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Get node performance metrics
|
|
203
|
-
* @returns {NodeMetrics} Node performance metrics
|
|
204
|
-
*/
|
|
205
|
-
getMetrics() {
|
|
206
|
-
return {
|
|
207
|
-
...this.metrics,
|
|
208
|
-
dependencies: this.getDependencies(),
|
|
209
|
-
successRate:
|
|
210
|
-
this.metrics.executionCount > 0
|
|
211
|
-
? this.metrics.successCount / this.metrics.executionCount
|
|
212
|
-
: 0,
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
/**
|
|
217
|
-
* Reset node metrics
|
|
218
|
-
*/
|
|
219
|
-
resetMetrics() {
|
|
220
|
-
this.metrics = {
|
|
221
|
-
executionCount: 0,
|
|
222
|
-
totalExecutionTime: 0,
|
|
223
|
-
successCount: 0,
|
|
224
|
-
failureCount: 0,
|
|
225
|
-
retryCount: 0,
|
|
226
|
-
lastExecutionTime: null,
|
|
227
|
-
avgExecutionTime: 0,
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/**
|
|
233
|
-
* Enhanced DAG Engine with advanced performance and reliability features
|
|
234
|
-
*/
|
|
235
|
-
class EnhancedDAGEngine {
|
|
236
|
-
/**
|
|
237
|
-
* Create a new enhanced DAG engine
|
|
238
|
-
* @param {Object} [options={}] - Engine configuration options
|
|
239
|
-
*/
|
|
240
|
-
constructor(options = {}) {
|
|
241
|
-
this.nodes = new Map();
|
|
242
|
-
this.executionHistory = [];
|
|
243
|
-
this.enableMetrics = options.enableMetrics !== false;
|
|
244
|
-
this.enableTracing = options.enableTracing || false;
|
|
245
|
-
|
|
246
|
-
// Default execution options
|
|
247
|
-
this.defaultOptions = {
|
|
248
|
-
timeout: 30000,
|
|
249
|
-
concurrency: parseInt(process.env.RAG_MAX_CONCURRENCY) || 5,
|
|
250
|
-
continueOnError: false,
|
|
251
|
-
gracefulDegradation: false,
|
|
252
|
-
retryFailedNodes: false,
|
|
253
|
-
maxRetries: 3,
|
|
254
|
-
enableMetrics: this.enableMetrics,
|
|
255
|
-
enableTracing: this.enableTracing,
|
|
256
|
-
};
|
|
257
|
-
|
|
258
|
-
// Performance tracking
|
|
259
|
-
this.engineMetrics = {
|
|
260
|
-
totalExecutions: 0,
|
|
261
|
-
successfulExecutions: 0,
|
|
262
|
-
failedExecutions: 0,
|
|
263
|
-
avgExecutionTime: 0,
|
|
264
|
-
totalExecutionTime: 0,
|
|
265
|
-
};
|
|
266
|
-
|
|
267
|
-
// Create engine logger
|
|
268
|
-
this.logger = logger.child({
|
|
269
|
-
component: 'enhanced-dag-engine',
|
|
270
|
-
version: '2.1.8',
|
|
271
|
-
});
|
|
272
|
-
|
|
273
|
-
this.logger.info('Enhanced DAG Engine initialized', {
|
|
274
|
-
defaultConcurrency: this.defaultOptions.concurrency,
|
|
275
|
-
metricsEnabled: this.enableMetrics,
|
|
276
|
-
tracingEnabled: this.enableTracing,
|
|
277
|
-
});
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
/**
|
|
281
|
-
* Add a node to the DAG
|
|
282
|
-
* @param {string} id - Unique node identifier
|
|
283
|
-
* @param {Function} run - Async function to execute
|
|
284
|
-
* @param {DAGNodeOptions} [options={}] - Node configuration
|
|
285
|
-
* @returns {EnhancedDAGNode} The created node
|
|
286
|
-
*/
|
|
287
|
-
addNode(id, run, options = {}) {
|
|
288
|
-
if (this.nodes.has(id)) {
|
|
289
|
-
throw new Error(`Node with ID '${id}' already exists`);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
const node = new EnhancedDAGNode(id, run, options);
|
|
293
|
-
this.nodes.set(id, node);
|
|
294
|
-
|
|
295
|
-
this.logger.debug('Node added to DAG', {
|
|
296
|
-
nodeId: id,
|
|
297
|
-
priority: node.priority,
|
|
298
|
-
optional: node.optional,
|
|
299
|
-
timeout: node.timeout,
|
|
300
|
-
});
|
|
301
|
-
|
|
302
|
-
return node;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
/**
|
|
306
|
-
* Get a node by ID
|
|
307
|
-
* @param {string} id - Node identifier
|
|
308
|
-
* @returns {EnhancedDAGNode|undefined} The node or undefined if not found
|
|
309
|
-
*/
|
|
310
|
-
getNode(id) {
|
|
311
|
-
return this.nodes.get(id);
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
/**
|
|
315
|
-
* Connect two nodes
|
|
316
|
-
* @param {string} fromId - Source node ID
|
|
317
|
-
* @param {string} toId - Target node ID
|
|
318
|
-
* @returns {EnhancedDAGEngine} This engine for chaining
|
|
319
|
-
*/
|
|
320
|
-
connect(fromId, toId) {
|
|
321
|
-
const fromNode = this.nodes.get(fromId);
|
|
322
|
-
const toNode = this.nodes.get(toId);
|
|
323
|
-
|
|
324
|
-
if (!fromNode) {
|
|
325
|
-
throw new Error(`Source node '${fromId}' not found`);
|
|
326
|
-
}
|
|
327
|
-
if (!toNode) {
|
|
328
|
-
throw new Error(`Target node '${toId}' not found`);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
fromNode.addOutput(toNode);
|
|
332
|
-
|
|
333
|
-
this.logger.debug('Nodes connected', {
|
|
334
|
-
fromNodeId: fromId,
|
|
335
|
-
toNodeId: toId,
|
|
336
|
-
});
|
|
337
|
-
|
|
338
|
-
return this;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* Validate DAG for cycles and structural integrity
|
|
343
|
-
* @throws {Error} If DAG is invalid
|
|
344
|
-
*/
|
|
345
|
-
validate() {
|
|
346
|
-
this.logger.debug('Validating DAG structure');
|
|
347
|
-
|
|
348
|
-
// Check for cycles using DFS
|
|
349
|
-
const visited = new Set();
|
|
350
|
-
const recursionStack = new Set();
|
|
351
|
-
|
|
352
|
-
const hasCycle = (nodeId) => {
|
|
353
|
-
if (recursionStack.has(nodeId)) {
|
|
354
|
-
return true;
|
|
355
|
-
}
|
|
356
|
-
if (visited.has(nodeId)) {
|
|
357
|
-
return false;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
visited.add(nodeId);
|
|
361
|
-
recursionStack.add(nodeId);
|
|
362
|
-
|
|
363
|
-
const node = this.nodes.get(nodeId);
|
|
364
|
-
for (const output of node.outputs) {
|
|
365
|
-
if (hasCycle(output.id)) {
|
|
366
|
-
return true;
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
recursionStack.delete(nodeId);
|
|
371
|
-
return false;
|
|
372
|
-
};
|
|
373
|
-
|
|
374
|
-
for (const nodeId of this.nodes.keys()) {
|
|
375
|
-
if (hasCycle(nodeId)) {
|
|
376
|
-
throw new Error(`Cycle detected in DAG involving node: ${nodeId}`);
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// Validate node integrity
|
|
381
|
-
for (const [nodeId, node] of this.nodes) {
|
|
382
|
-
// Check that all input/output connections are bidirectional
|
|
383
|
-
for (const input of node.inputs) {
|
|
384
|
-
if (!input.outputs.includes(node)) {
|
|
385
|
-
throw new Error(
|
|
386
|
-
`Broken bidirectional connection: ${input.id} -> ${nodeId}`,
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
for (const output of node.outputs) {
|
|
391
|
-
if (!output.inputs.includes(node)) {
|
|
392
|
-
throw new Error(
|
|
393
|
-
`Broken bidirectional connection: ${nodeId} -> ${output.id}`,
|
|
394
|
-
);
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
this.logger.info('DAG validation completed successfully', {
|
|
400
|
-
nodeCount: this.nodes.size,
|
|
401
|
-
});
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
/**
|
|
405
|
-
* Get topological order of nodes for execution
|
|
406
|
-
* @returns {EnhancedDAGNode[]} Nodes in execution order
|
|
407
|
-
*/
|
|
408
|
-
getTopologicalOrder() {
|
|
409
|
-
const visited = new Set();
|
|
410
|
-
const order = [];
|
|
411
|
-
|
|
412
|
-
const visit = (node) => {
|
|
413
|
-
if (visited.has(node.id)) {
|
|
414
|
-
return;
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
visited.add(node.id);
|
|
418
|
-
|
|
419
|
-
// Visit dependencies first (sorted by priority)
|
|
420
|
-
const sortedInputs = [...node.inputs].sort(
|
|
421
|
-
(a, b) => b.priority - a.priority,
|
|
422
|
-
);
|
|
423
|
-
for (const input of sortedInputs) {
|
|
424
|
-
visit(input);
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
order.push(node);
|
|
428
|
-
};
|
|
429
|
-
|
|
430
|
-
// Start with nodes that have no inputs, sorted by priority
|
|
431
|
-
const sourceNodes = Array.from(this.nodes.values())
|
|
432
|
-
.filter((node) => node.inputs.length === 0)
|
|
433
|
-
.sort((a, b) => b.priority - a.priority);
|
|
434
|
-
|
|
435
|
-
for (const node of sourceNodes) {
|
|
436
|
-
visit(node);
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
// Ensure all nodes are included
|
|
440
|
-
for (const node of this.nodes.values()) {
|
|
441
|
-
visit(node);
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
this.logger.debug('Topological order computed', {
|
|
445
|
-
nodeCount: order.length,
|
|
446
|
-
sourceNodes: sourceNodes.map((n) => n.id),
|
|
447
|
-
});
|
|
448
|
-
|
|
449
|
-
return order;
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
/**
|
|
453
|
-
* Execute a single node with comprehensive error handling and retry logic
|
|
454
|
-
* @param {EnhancedDAGNode} node - Node to execute
|
|
455
|
-
* @param {ExecutionContext} context - Execution context
|
|
456
|
-
* @returns {Promise<any>} Node execution result
|
|
457
|
-
*/
|
|
458
|
-
async executeNode(node, context) {
|
|
459
|
-
const { results, errors, seed, maxRetries } = context;
|
|
460
|
-
const startTime = Date.now();
|
|
461
|
-
let retryCount = 0;
|
|
462
|
-
|
|
463
|
-
const correlationId = this.logger.getContext()?.correlationId;
|
|
464
|
-
|
|
465
|
-
node.logger.debug('Starting node execution', {
|
|
466
|
-
correlationId,
|
|
467
|
-
dependencies: node.getDependencies(),
|
|
468
|
-
hasResults: node.inputs.map((dep) => results.has(dep.id)),
|
|
469
|
-
});
|
|
470
|
-
|
|
471
|
-
try {
|
|
472
|
-
// Prepare input data from dependencies
|
|
473
|
-
let input = seed;
|
|
474
|
-
if (node.inputs.length > 0) {
|
|
475
|
-
input = {};
|
|
476
|
-
for (const dep of node.inputs) {
|
|
477
|
-
if (results.has(dep.id)) {
|
|
478
|
-
input[dep.id] = results.get(dep.id);
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
// Create execution function with timeout
|
|
484
|
-
const executeWithTimeout = async () => {
|
|
485
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
486
|
-
setTimeout(() => {
|
|
487
|
-
reject(
|
|
488
|
-
new Error(
|
|
489
|
-
`Node ${node.id} execution timeout after ${node.timeout}ms`,
|
|
490
|
-
),
|
|
491
|
-
);
|
|
492
|
-
}, node.timeout);
|
|
493
|
-
});
|
|
494
|
-
|
|
495
|
-
return Promise.race([node.run(input), timeoutPromise]);
|
|
496
|
-
};
|
|
497
|
-
|
|
498
|
-
// Use retry policy if configured
|
|
499
|
-
let result;
|
|
500
|
-
if (node.retryPolicy || (node.retries > 0 && maxRetries > 0)) {
|
|
501
|
-
const retryPolicy = node.retryPolicy || defaultRetryPolicy;
|
|
502
|
-
|
|
503
|
-
result = await retryPolicy.execute(executeWithTimeout, {
|
|
504
|
-
nodeId: node.id,
|
|
505
|
-
correlationId,
|
|
506
|
-
});
|
|
507
|
-
retryCount = retryPolicy.getMetrics().retryBudget.currentRetries;
|
|
508
|
-
} else {
|
|
509
|
-
result = await executeWithTimeout();
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
const executionTime = Date.now() - startTime;
|
|
513
|
-
|
|
514
|
-
// Update metrics
|
|
515
|
-
if (this.enableMetrics) {
|
|
516
|
-
node.updateMetrics(executionTime, true, retryCount);
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
node.logger.info('Node execution completed successfully', {
|
|
520
|
-
correlationId,
|
|
521
|
-
executionTime,
|
|
522
|
-
retryCount,
|
|
523
|
-
resultType: typeof result,
|
|
524
|
-
});
|
|
525
|
-
|
|
526
|
-
return result;
|
|
527
|
-
} catch (error) {
|
|
528
|
-
const executionTime = Date.now() - startTime;
|
|
529
|
-
|
|
530
|
-
// Update metrics
|
|
531
|
-
if (this.enableMetrics) {
|
|
532
|
-
node.updateMetrics(executionTime, false, retryCount);
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
// Handle optional node failures
|
|
536
|
-
if (node.optional && context.gracefulDegradation) {
|
|
537
|
-
node.logger.warn(
|
|
538
|
-
'Optional node failed, continuing with graceful degradation',
|
|
539
|
-
{
|
|
540
|
-
correlationId,
|
|
541
|
-
error: error.message,
|
|
542
|
-
executionTime,
|
|
543
|
-
retryCount,
|
|
544
|
-
},
|
|
545
|
-
);
|
|
546
|
-
return Symbol('failed-optional-node');
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
// Store error for required nodes
|
|
550
|
-
errors.set(node.id, error);
|
|
551
|
-
|
|
552
|
-
node.logger.error('Node execution failed', {
|
|
553
|
-
correlationId,
|
|
554
|
-
error: error.message,
|
|
555
|
-
errorCode: error.code,
|
|
556
|
-
executionTime,
|
|
557
|
-
retryCount,
|
|
558
|
-
stack: error.stack,
|
|
559
|
-
});
|
|
560
|
-
|
|
561
|
-
// Re-throw for required nodes
|
|
562
|
-
throw error;
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
/**
|
|
567
|
-
* Execute the DAG with advanced concurrency and error handling
|
|
568
|
-
* @param {any} [seed] - Initial seed value for source nodes
|
|
569
|
-
* @param {ExecutionOptions} [options={}] - Execution options
|
|
570
|
-
* @returns {Promise<Object>} Execution results
|
|
571
|
-
*/
|
|
572
|
-
async execute(seed, options = {}) {
|
|
573
|
-
const executionId = require('crypto').randomUUID();
|
|
574
|
-
const startTime = Date.now();
|
|
575
|
-
|
|
576
|
-
// Merge options with defaults
|
|
577
|
-
const config = { ...this.defaultOptions, ...options };
|
|
578
|
-
|
|
579
|
-
return this.logger.withCorrelation(executionId, async () => {
|
|
580
|
-
this.logger.info('Starting DAG execution', {
|
|
581
|
-
executionId,
|
|
582
|
-
seedType: typeof seed,
|
|
583
|
-
nodeCount: this.nodes.size,
|
|
584
|
-
concurrency: config.concurrency,
|
|
585
|
-
timeout: config.timeout,
|
|
586
|
-
});
|
|
587
|
-
|
|
588
|
-
try {
|
|
589
|
-
// Validate DAG before execution
|
|
590
|
-
this.validate();
|
|
591
|
-
|
|
592
|
-
// Initialize execution state and context
|
|
593
|
-
const results = new Map();
|
|
594
|
-
const errors = new Map();
|
|
595
|
-
const order = this.getTopologicalOrder();
|
|
596
|
-
const context = this._buildExecutionContext(
|
|
597
|
-
results,
|
|
598
|
-
errors,
|
|
599
|
-
seed,
|
|
600
|
-
config,
|
|
601
|
-
);
|
|
602
|
-
|
|
603
|
-
// Schedule and run all nodes with concurrency control
|
|
604
|
-
await this._scheduleNodes(order, context, config);
|
|
605
|
-
|
|
606
|
-
const executionTime = Date.now() - startTime;
|
|
607
|
-
|
|
608
|
-
// Update engine metrics
|
|
609
|
-
this._recordExecutionMetrics(executionTime, errors.size === 0);
|
|
610
|
-
|
|
611
|
-
// Build and store execution summary
|
|
612
|
-
const summary = this._buildExecutionSummary(
|
|
613
|
-
executionId,
|
|
614
|
-
executionTime,
|
|
615
|
-
results,
|
|
616
|
-
errors,
|
|
617
|
-
);
|
|
618
|
-
this._storeExecutionHistory(summary, results);
|
|
619
|
-
|
|
620
|
-
this.logger.info('DAG execution completed', summary);
|
|
621
|
-
|
|
622
|
-
return {
|
|
623
|
-
results: Object.fromEntries(results),
|
|
624
|
-
errors: Object.fromEntries(errors),
|
|
625
|
-
summary,
|
|
626
|
-
metrics: this.enableMetrics ? this.getMetrics() : null,
|
|
627
|
-
};
|
|
628
|
-
} catch (error) {
|
|
629
|
-
const executionTime = Date.now() - startTime;
|
|
630
|
-
this._recordExecutionFailure(executionId, error, executionTime);
|
|
631
|
-
throw error;
|
|
632
|
-
}
|
|
633
|
-
});
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
/**
|
|
637
|
-
* Build the execution context passed to each node during DAG execution.
|
|
638
|
-
* @param {Map} results - Map to store completed node results
|
|
639
|
-
* @param {Map} errors - Map to store node execution errors
|
|
640
|
-
* @param {any} seed - Initial seed value for source nodes
|
|
641
|
-
* @param {Object} config - Merged execution configuration
|
|
642
|
-
* @returns {ExecutionContext} Execution context object
|
|
643
|
-
* @private
|
|
644
|
-
*/
|
|
645
|
-
_buildExecutionContext(results, errors, seed, config) {
|
|
646
|
-
return {
|
|
647
|
-
results,
|
|
648
|
-
errors,
|
|
649
|
-
seed,
|
|
650
|
-
continueOnError: config.continueOnError,
|
|
651
|
-
gracefulDegradation: config.gracefulDegradation,
|
|
652
|
-
retryFailedNodes: config.retryFailedNodes,
|
|
653
|
-
maxRetries: config.maxRetries,
|
|
654
|
-
};
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
/**
|
|
658
|
-
* Schedule and execute all nodes in topological order with concurrency control.
|
|
659
|
-
* Each node waits for its dependencies before executing.
|
|
660
|
-
* @param {EnhancedDAGNode[]} order - Nodes in topological execution order
|
|
661
|
-
* @param {ExecutionContext} context - Execution context with results and errors maps
|
|
662
|
-
* @param {Object} config - Execution configuration including concurrency limit
|
|
663
|
-
* @returns {Promise<void>} Resolves when all nodes have settled
|
|
664
|
-
* @private
|
|
665
|
-
*/
|
|
666
|
-
async _scheduleNodes(order, context, config) {
|
|
667
|
-
const { results } = context;
|
|
668
|
-
const semaphore = new Semaphore(config.concurrency);
|
|
669
|
-
const nodePromises = new Map();
|
|
670
|
-
|
|
671
|
-
for (const node of order) {
|
|
672
|
-
const promise = semaphore.acquire().then(async (release) => {
|
|
673
|
-
try {
|
|
674
|
-
// Wait for dependencies to complete
|
|
675
|
-
const depPromises = node.inputs
|
|
676
|
-
.map((dep) => nodePromises.get(dep.id))
|
|
677
|
-
.filter(Boolean);
|
|
678
|
-
|
|
679
|
-
await Promise.all(depPromises);
|
|
680
|
-
|
|
681
|
-
// Execute the node
|
|
682
|
-
const result = await this.executeNode(node, context);
|
|
683
|
-
|
|
684
|
-
if (
|
|
685
|
-
typeof result !== 'symbol' ||
|
|
686
|
-
result.description !== 'failed-optional-node'
|
|
687
|
-
) {
|
|
688
|
-
results.set(node.id, result);
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
return result;
|
|
692
|
-
} finally {
|
|
693
|
-
release();
|
|
694
|
-
}
|
|
695
|
-
});
|
|
696
|
-
|
|
697
|
-
nodePromises.set(node.id, promise);
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
await Promise.allSettled(Array.from(nodePromises.values()));
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
/**
|
|
704
|
-
* Record engine-level metrics after a completed execution.
|
|
705
|
-
* @param {number} executionTime - Total execution time in milliseconds
|
|
706
|
-
* @param {boolean} success - Whether the execution completed without errors
|
|
707
|
-
* @private
|
|
708
|
-
*/
|
|
709
|
-
_recordExecutionMetrics(executionTime, success) {
|
|
710
|
-
if (!this.enableMetrics) {
|
|
711
|
-
return;
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
this.engineMetrics.totalExecutions++;
|
|
715
|
-
this.engineMetrics.totalExecutionTime += executionTime;
|
|
716
|
-
this.engineMetrics.avgExecutionTime =
|
|
717
|
-
this.engineMetrics.totalExecutionTime /
|
|
718
|
-
this.engineMetrics.totalExecutions;
|
|
719
|
-
|
|
720
|
-
if (success) {
|
|
721
|
-
this.engineMetrics.successfulExecutions++;
|
|
722
|
-
} else {
|
|
723
|
-
this.engineMetrics.failedExecutions++;
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
/**
|
|
728
|
-
* Build a summary object describing the outcome of a DAG execution.
|
|
729
|
-
* @param {string} executionId - Unique execution identifier
|
|
730
|
-
* @param {number} executionTime - Total execution time in milliseconds
|
|
731
|
-
* @param {Map} results - Map of successful node results
|
|
732
|
-
* @param {Map} errors - Map of node execution errors
|
|
733
|
-
* @returns {Object} Execution summary
|
|
734
|
-
* @private
|
|
735
|
-
*/
|
|
736
|
-
_buildExecutionSummary(executionId, executionTime, results, errors) {
|
|
737
|
-
return {
|
|
738
|
-
executionId,
|
|
739
|
-
success: errors.size === 0,
|
|
740
|
-
executionTime,
|
|
741
|
-
nodesExecuted: results.size,
|
|
742
|
-
totalNodes: this.nodes.size,
|
|
743
|
-
errors: Array.from(errors.entries()).map(([id, error]) => ({
|
|
744
|
-
nodeId: id,
|
|
745
|
-
message: error.message,
|
|
746
|
-
code: error.code,
|
|
747
|
-
})),
|
|
748
|
-
};
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
/**
|
|
752
|
-
* Store an execution summary in the tracing history buffer.
|
|
753
|
-
* Keeps only the most recent 100 entries.
|
|
754
|
-
* @param {Object} summary - Execution summary to store
|
|
755
|
-
* @param {Map} results - Map of node results to include in history
|
|
756
|
-
* @private
|
|
757
|
-
*/
|
|
758
|
-
_storeExecutionHistory(summary, results) {
|
|
759
|
-
if (!this.enableTracing) {
|
|
760
|
-
return;
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
this.executionHistory.push({
|
|
764
|
-
...summary,
|
|
765
|
-
timestamp: new Date().toISOString(),
|
|
766
|
-
results: Object.fromEntries(results),
|
|
767
|
-
});
|
|
768
|
-
|
|
769
|
-
if (this.executionHistory.length > 100) {
|
|
770
|
-
this.executionHistory.shift();
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
/**
|
|
775
|
-
* Record metrics and log details when a DAG execution fails with an exception.
|
|
776
|
-
* @param {string} executionId - Unique execution identifier
|
|
777
|
-
* @param {Error} error - The error that caused the failure
|
|
778
|
-
* @param {number} executionTime - Time elapsed before the failure
|
|
779
|
-
* @private
|
|
780
|
-
*/
|
|
781
|
-
_recordExecutionFailure(executionId, error, executionTime) {
|
|
782
|
-
if (this.enableMetrics) {
|
|
783
|
-
this.engineMetrics.totalExecutions++;
|
|
784
|
-
this.engineMetrics.failedExecutions++;
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
this.logger.error('DAG execution failed', {
|
|
788
|
-
executionId,
|
|
789
|
-
error: error.message,
|
|
790
|
-
executionTime,
|
|
791
|
-
stack: error.stack,
|
|
792
|
-
});
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
/**
|
|
796
|
-
* Get comprehensive performance metrics
|
|
797
|
-
* @returns {Object} Performance metrics
|
|
798
|
-
*/
|
|
799
|
-
getMetrics() {
|
|
800
|
-
const nodeMetrics = {};
|
|
801
|
-
for (const [id, node] of this.nodes) {
|
|
802
|
-
nodeMetrics[id] = node.getMetrics();
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
return {
|
|
806
|
-
engine: this.engineMetrics,
|
|
807
|
-
nodes: nodeMetrics,
|
|
808
|
-
summary: {
|
|
809
|
-
totalNodes: this.nodes.size,
|
|
810
|
-
avgSuccessRate:
|
|
811
|
-
Object.values(nodeMetrics).reduce(
|
|
812
|
-
(sum, metrics) => sum + metrics.successRate,
|
|
813
|
-
0,
|
|
814
|
-
) / this.nodes.size,
|
|
815
|
-
},
|
|
816
|
-
};
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
/**
|
|
820
|
-
* Reset all metrics
|
|
821
|
-
*/
|
|
822
|
-
resetMetrics() {
|
|
823
|
-
this.engineMetrics = {
|
|
824
|
-
totalExecutions: 0,
|
|
825
|
-
successfulExecutions: 0,
|
|
826
|
-
failedExecutions: 0,
|
|
827
|
-
avgExecutionTime: 0,
|
|
828
|
-
totalExecutionTime: 0,
|
|
829
|
-
};
|
|
830
|
-
|
|
831
|
-
for (const node of this.nodes.values()) {
|
|
832
|
-
node.resetMetrics();
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
this.executionHistory = [];
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
/**
|
|
839
|
-
* Get execution history (if tracing is enabled)
|
|
840
|
-
* @returns {Array} Execution history
|
|
841
|
-
*/
|
|
842
|
-
getExecutionHistory() {
|
|
843
|
-
return [...this.executionHistory];
|
|
844
|
-
}
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
/**
|
|
848
|
-
* Semaphore for concurrency control
|
|
849
|
-
*/
|
|
850
|
-
class Semaphore {
|
|
851
|
-
constructor(maxConcurrency) {
|
|
852
|
-
this.maxConcurrency = maxConcurrency;
|
|
853
|
-
this.currentCount = 0;
|
|
854
|
-
this.waitQueue = [];
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
async acquire() {
|
|
858
|
-
return new Promise((resolve) => {
|
|
859
|
-
if (this.currentCount < this.maxConcurrency) {
|
|
860
|
-
this.currentCount++;
|
|
861
|
-
resolve(() => this.release());
|
|
862
|
-
} else {
|
|
863
|
-
this.waitQueue.push(() => {
|
|
864
|
-
this.currentCount++;
|
|
865
|
-
resolve(() => this.release());
|
|
866
|
-
});
|
|
867
|
-
}
|
|
868
|
-
});
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
release() {
|
|
872
|
-
this.currentCount--;
|
|
873
|
-
if (this.waitQueue.length > 0) {
|
|
874
|
-
const next = this.waitQueue.shift();
|
|
875
|
-
next();
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
/**
|
|
881
|
-
* Create enhanced DAG engine instance
|
|
882
|
-
* @param {Object} [options={}] - Engine configuration
|
|
883
|
-
* @returns {EnhancedDAGEngine} DAG engine instance
|
|
884
|
-
*/
|
|
885
|
-
function createEnhancedDAG(options = {}) {
|
|
886
|
-
return new EnhancedDAGEngine(options);
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
module.exports = {
|
|
890
|
-
EnhancedDAGEngine,
|
|
891
|
-
EnhancedDAGNode,
|
|
892
|
-
Semaphore,
|
|
893
|
-
createEnhancedDAG,
|
|
894
|
-
};
|