@devilsdev/rag-pipeline-utils 2.1.12 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.ragrc.schema.json +218 -0
  2. package/README.md +292 -32
  3. package/bin/cli.js +8 -14
  4. package/dist/index.d.ts +237 -0
  5. package/dist/index.mjs +22 -7
  6. package/package.json +17 -13
  7. package/src/ai/adaptive-retrieval.js +191 -0
  8. package/src/ai/federated-learning.js +214 -0
  9. package/src/ai/federation/federated-learning-coordinator.js +345 -0
  10. package/src/ai/federation.js +76 -0
  11. package/src/ai/index-fixed.js +509 -0
  12. package/src/ai/index.js +33 -0
  13. package/src/ai/model-trainer.js +217 -0
  14. package/src/ai/model-training.js +205 -0
  15. package/src/ai/module-template.js +37 -0
  16. package/src/ai/multimodal/multi-modal-processor.js +345 -0
  17. package/src/ai/multimodal-processing.js +268 -0
  18. package/src/ai/multimodal.js +71 -0
  19. package/src/ai/orchestrator.js +70 -0
  20. package/src/ai/retrieval/adaptive-retrieval-engine.js +398 -0
  21. package/src/ai/retrieval-engine.js +69 -0
  22. package/src/ai/training/model-training-orchestrator.js +228 -0
  23. package/src/ci/workflow-analyzer.js +71 -0
  24. package/src/cli/commands/ai-ml.js +967 -0
  25. package/src/cli/commands/docs.js +228 -0
  26. package/src/cli/commands/dx.js +538 -0
  27. package/src/cli/commands/plugin-hub.js +875 -0
  28. package/src/cli/doctor-command.js +639 -0
  29. package/src/cli/enhanced-cli-commands.js +945 -0
  30. package/src/cli/interactive-wizard.js +997 -0
  31. package/src/cli/plugin-marketplace-commands.js +869 -0
  32. package/src/cli.js +17 -0
  33. package/src/config/enhanced-ragrc-schema.js +632 -0
  34. package/src/config/load-config.js +54 -0
  35. package/src/config/load-plugin-config.js +69 -0
  36. package/src/config/normalize-config.js +105 -0
  37. package/src/config/schema-validator.js +315 -0
  38. package/src/config/validate-plugin-schema.js +35 -0
  39. package/src/config/validate-schema.js +113 -0
  40. package/src/contracts/plugin-contracts.js +33 -0
  41. package/src/core/create-pipeline.js +99 -0
  42. package/src/core/dependency-injection.js +242 -0
  43. package/src/core/observability/event-logger.js +241 -0
  44. package/src/core/observability/instrumented-pipeline.js +609 -0
  45. package/src/core/observability/metrics.js +497 -0
  46. package/src/core/observability/tracing.js +781 -0
  47. package/src/core/performance/benchmark.js +516 -0
  48. package/src/core/performance/parallel-processor.js +240 -0
  49. package/src/core/performance/streaming-safeguards.js +611 -0
  50. package/src/core/pipeline-factory.js +60 -0
  51. package/src/core/plugin-contracts.js +89 -0
  52. package/src/core/plugin-marketplace/plugin-metadata.js +485 -0
  53. package/src/core/plugin-marketplace/plugin-publisher.js +574 -0
  54. package/src/core/plugin-marketplace/plugin-registry-format.js +419 -0
  55. package/src/core/plugin-marketplace/version-resolver.js +61 -0
  56. package/src/core/plugin-registry.js +265 -0
  57. package/src/dag/dag-engine.js +954 -0
  58. package/src/dag/dag-node.js +76 -0
  59. package/src/dag/enhanced-dag-engine.js +822 -0
  60. package/src/dx/index.js +40 -0
  61. package/src/dx/integration-templates.js +665 -0
  62. package/src/dx/performance-profiler.js +968 -0
  63. package/src/dx/realtime-debugger.js +755 -0
  64. package/src/dx/visual-pipeline-builder.js +578 -0
  65. package/src/ecosystem/plugin-analytics-dashboard.js +778 -0
  66. package/src/ecosystem/plugin-certification.js +580 -0
  67. package/src/ecosystem/plugin-hub.js +821 -0
  68. package/src/enterprise/audit-logging.js +967 -0
  69. package/src/enterprise/data-governance.js +408 -0
  70. package/src/enterprise/multi-tenancy.js +496 -0
  71. package/src/enterprise/sso-integration.js +903 -0
  72. package/src/evaluate/evaluator.js +52 -0
  73. package/src/evaluate/scoring.js +91 -0
  74. package/src/index.js +67 -0
  75. package/src/ingest.js +59 -0
  76. package/src/loader/csv-loader.js +38 -0
  77. package/src/loader/directory-loader.js +48 -0
  78. package/src/loader/html-loader.js +62 -0
  79. package/src/loader/markdown-loader.js +67 -0
  80. package/src/mocks/openai-embedder.js +119 -0
  81. package/src/mocks/openai-llm.js +77 -0
  82. package/src/mocks/pdf-loader.js +21 -0
  83. package/src/mocks/pinecone-retriever.js +106 -0
  84. package/src/observability/audit-logger.js +758 -0
  85. package/src/observability/dashboard-config.json +675 -0
  86. package/src/observability/event-logger.js +310 -0
  87. package/src/observability/metrics.js +691 -0
  88. package/src/observability/security-monitor.js +830 -0
  89. package/src/observability/slo-monitor.js +319 -0
  90. package/src/observability/telemetry.js +686 -0
  91. package/src/pipeline/pipeline.js +18 -0
  92. package/src/plugins/sample-embedder.js +53 -0
  93. package/src/plugins/sample-llm.js +30 -0
  94. package/src/plugins/sample-loader.js +24 -0
  95. package/src/plugins/sample-retriever.js +84 -0
  96. package/src/query.js +117 -0
  97. package/src/reranker/llm-reranker.js +85 -0
  98. package/src/scripts/script-utilities.js +36 -0
  99. package/src/security/plugin-sandbox.js +586 -0
  100. package/src/security/plugin-signature-verifier.js +414 -0
  101. package/src/security/validator.js +36 -0
  102. package/src/utils/ci/diagnostic-reporter.js +57 -0
  103. package/src/utils/logger.js +211 -0
  104. package/src/utils/performance-monitor.js +561 -0
  105. package/src/utils/plugin-scaffolder.js +447 -0
  106. package/src/utils/retry-policy.js +443 -0
  107. package/src/utils/retry.js +107 -0
  108. package/src/utils/safe-expression-evaluator.js +579 -0
  109. package/src/utils/structured-logger.js +361 -0
  110. package/src/utils/validate-plugin-contract.js +42 -0
@@ -0,0 +1,265 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+ const {
6
+ createPluginVerifier,
7
+ } = require('../security/plugin-signature-verifier.js');
8
+
9
+ class PluginRegistry {
10
+ constructor(options = {}) {
11
+ this._plugins = new Map();
12
+ this._validTypes = new Set([
13
+ 'loader',
14
+ 'embedder',
15
+ 'retriever',
16
+ 'reranker',
17
+ 'llm',
18
+ 'evaluator',
19
+ ]);
20
+ this._contracts = new Map();
21
+
22
+ // Environment-aware signature verification defaults
23
+ const isDevelopment = process.env.NODE_ENV === 'development';
24
+ const isProduction = process.env.NODE_ENV === 'production';
25
+
26
+ this._signatureVerifier = createPluginVerifier({
27
+ enabled:
28
+ options.verifySignatures !== undefined
29
+ ? options.verifySignatures
30
+ : isProduction, // Enabled by default in production, disabled in development
31
+ failClosed:
32
+ options.failClosed !== undefined ? options.failClosed : isProduction, // Fail closed in production, fail open in development
33
+ trustedKeysPath: options.trustedKeysPath,
34
+ });
35
+ this._loadContracts();
36
+ }
37
+
38
+ _loadContracts() {
39
+ const contractsDir = path.join(__dirname, '../../contracts');
40
+
41
+ try {
42
+ const contractFiles = {
43
+ loader: 'loader-contract.json',
44
+ embedder: 'embedder-contract.json',
45
+ retriever: 'retriever-contract.json',
46
+ llm: 'llm-contract.json',
47
+ };
48
+
49
+ for (const [type, filename] of Object.entries(contractFiles)) {
50
+ try {
51
+ const contractPath = path.join(contractsDir, filename);
52
+ if (fs.existsSync(contractPath)) {
53
+ const contractContent = fs.readFileSync(contractPath, 'utf8');
54
+ const contract = JSON.parse(contractContent);
55
+ this._contracts.set(type, contract);
56
+ }
57
+ } catch (error) {
58
+ // Contract loading is optional - don't fail if missing
59
+ }
60
+ }
61
+ } catch (error) {
62
+ // Contracts directory may not exist in all environments
63
+ }
64
+ }
65
+
66
+ _validatePluginContract(category, impl) {
67
+ const contract = this._contracts.get(category);
68
+ if (!contract) {
69
+ return; // No contract to validate against
70
+ }
71
+
72
+ // Check if plugin has metadata
73
+ if (!impl.metadata) {
74
+ throw new Error('Plugin missing metadata property');
75
+ }
76
+
77
+ if (!impl.metadata.name || typeof impl.metadata.name !== 'string') {
78
+ throw new Error('Plugin metadata must have a name property');
79
+ }
80
+
81
+ if (!impl.metadata.version || typeof impl.metadata.version !== 'string') {
82
+ throw new Error('Plugin metadata must have a version property');
83
+ }
84
+
85
+ if (!impl.metadata.type || impl.metadata.type !== category) {
86
+ throw new Error(`Plugin metadata type must be '${category}'`);
87
+ }
88
+
89
+ // Validate required methods based on contract
90
+ const requiredMethods = this._getRequiredMethodsFromContract(contract);
91
+
92
+ for (const methodName of requiredMethods) {
93
+ if (!impl[methodName]) {
94
+ throw new Error(`Plugin missing required method: ${methodName}`);
95
+ }
96
+
97
+ if (typeof impl[methodName] !== 'function') {
98
+ throw new Error(`Plugin method '${methodName}' must be a function`);
99
+ }
100
+ }
101
+ }
102
+
103
+ _getRequiredMethodsFromContract(contract) {
104
+ const methods = [];
105
+
106
+ if (contract.required && Array.isArray(contract.required)) {
107
+ methods.push(...contract.required);
108
+ }
109
+
110
+ if (contract.properties) {
111
+ for (const [methodName, methodSpec] of Object.entries(
112
+ contract.properties,
113
+ )) {
114
+ if (methodSpec.type === 'function') {
115
+ methods.push(methodName);
116
+ }
117
+ }
118
+ }
119
+
120
+ return methods;
121
+ }
122
+
123
+ async register(category, name, impl, manifest = null) {
124
+ if (!category || typeof category !== 'string') {
125
+ throw new Error('Category must be a non-empty string');
126
+ }
127
+ if (!name || typeof name !== 'string') {
128
+ throw new Error('Name must be a non-empty string');
129
+ }
130
+ if (impl == null) {
131
+ throw new Error('Implementation cannot be null or undefined');
132
+ }
133
+ if (!this._validTypes.has(category)) {
134
+ throw new Error(`Unknown plugin type: ${category}`);
135
+ }
136
+
137
+ // Verify plugin signature if manifest provided
138
+ if (manifest) {
139
+ await this._verifyPluginSignature(manifest, name);
140
+ }
141
+
142
+ // Validate plugin against contract
143
+ this._validatePluginContract(category, impl);
144
+
145
+ const key = `${category}:${name}`;
146
+ this._plugins.set(key, impl);
147
+ return this;
148
+ }
149
+
150
+ /**
151
+ * Verify plugin signature
152
+ * @param {object} manifest - Plugin manifest
153
+ * @param {string} pluginName - Plugin name
154
+ */
155
+ async _verifyPluginSignature(manifest, pluginName) {
156
+ try {
157
+ // Extract signature information from manifest
158
+ if (!manifest.signature || !manifest.signerId) {
159
+ throw new Error(
160
+ `Plugin ${pluginName} missing required signature or signerId`,
161
+ );
162
+ }
163
+
164
+ const verificationResult =
165
+ await this._signatureVerifier.verifyPluginSignature(
166
+ manifest,
167
+ manifest.signature,
168
+ manifest.signerId,
169
+ );
170
+
171
+ if (!verificationResult.verified) {
172
+ throw new Error(
173
+ `Plugin ${pluginName} signature verification failed: ${verificationResult.error}`,
174
+ );
175
+ }
176
+
177
+ // Emit audit entry for successful verification
178
+ this._emitAuditEntry('plugin_verified', {
179
+ pluginName,
180
+ signerId: manifest.signerId,
181
+ version: manifest.version,
182
+ verified: true,
183
+ timestamp: new Date().toISOString(),
184
+ });
185
+ } catch (error) {
186
+ // Emit audit entry for failed verification
187
+ this._emitAuditEntry('plugin_verification_failed', {
188
+ pluginName,
189
+ signerId: manifest.signerId || 'unknown',
190
+ version: manifest.version || 'unknown',
191
+ verified: false,
192
+ error: error.message,
193
+ timestamp: new Date().toISOString(),
194
+ });
195
+
196
+ throw error;
197
+ }
198
+ }
199
+
200
+ /**
201
+ * Emit audit entry
202
+ * @param {string} action - Audit action
203
+ * @param {object} details - Audit details
204
+ */
205
+ _emitAuditEntry(action, details) {
206
+ const auditEntry = {
207
+ timestamp: new Date().toISOString(),
208
+ action,
209
+ component: 'plugin_registry',
210
+ ...details,
211
+ };
212
+
213
+ // In production, this should go to a secure audit log
214
+ if (process.env.NODE_ENV === 'production') {
215
+ console.warn('[AUDIT]', JSON.stringify(auditEntry)); // eslint-disable-line no-console
216
+ } else {
217
+ console.log('[AUDIT]', auditEntry); // eslint-disable-line no-console
218
+ }
219
+ }
220
+
221
+ get(category, name) {
222
+ const key = `${category}:${name}`;
223
+ const impl = this._plugins.get(key);
224
+ if (impl === undefined) {
225
+ throw new Error(`Plugin not found: ${category}/${name}`);
226
+ }
227
+ return impl;
228
+ }
229
+
230
+ resolve(category, name) {
231
+ return this.get(category, name);
232
+ }
233
+
234
+ list(category) {
235
+ if (!this._validTypes.has(category)) {
236
+ throw new Error(`Unknown plugin type: ${category}`);
237
+ }
238
+
239
+ const names = [];
240
+ const prefix = `${category}:`;
241
+
242
+ for (const key of this._plugins.keys()) {
243
+ if (key.startsWith(prefix)) {
244
+ names.push(key.substring(prefix.length));
245
+ }
246
+ }
247
+
248
+ return names;
249
+ }
250
+
251
+ clear() {
252
+ this._plugins.clear();
253
+ }
254
+ }
255
+
256
+ // Create singleton instance with default security settings
257
+ const registry = new PluginRegistry({
258
+ verifySignatures: process.env.NODE_ENV === 'production',
259
+ failClosed: process.env.NODE_ENV === 'production',
260
+ });
261
+
262
+ // CJS+ESM interop pattern
263
+ module.exports = registry;
264
+ module.exports.PluginRegistry = PluginRegistry;
265
+ module.exports.default = module.exports;