@devilsdev/rag-pipeline-utils 2.1.11 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ragrc.schema.json +218 -0
- package/README.md +651 -393
- package/bin/cli.js +8 -14
- package/dist/index.d.ts +237 -0
- package/dist/index.mjs +22 -7
- package/package.json +17 -13
- package/src/ai/adaptive-retrieval.js +191 -0
- package/src/ai/federated-learning.js +214 -0
- package/src/ai/federation/federated-learning-coordinator.js +345 -0
- package/src/ai/federation.js +76 -0
- package/src/ai/index-fixed.js +509 -0
- package/src/ai/index.js +33 -0
- package/src/ai/model-trainer.js +217 -0
- package/src/ai/model-training.js +205 -0
- package/src/ai/module-template.js +37 -0
- package/src/ai/multimodal/multi-modal-processor.js +345 -0
- package/src/ai/multimodal-processing.js +268 -0
- package/src/ai/multimodal.js +71 -0
- package/src/ai/orchestrator.js +70 -0
- package/src/ai/retrieval/adaptive-retrieval-engine.js +398 -0
- package/src/ai/retrieval-engine.js +69 -0
- package/src/ai/training/model-training-orchestrator.js +228 -0
- package/src/ci/workflow-analyzer.js +71 -0
- package/src/cli/commands/ai-ml.js +967 -0
- package/src/cli/commands/docs.js +228 -0
- package/src/cli/commands/dx.js +538 -0
- package/src/cli/commands/plugin-hub.js +875 -0
- package/src/cli/doctor-command.js +639 -0
- package/src/cli/enhanced-cli-commands.js +945 -0
- package/src/cli/interactive-wizard.js +997 -0
- package/src/cli/plugin-marketplace-commands.js +869 -0
- package/src/cli.js +17 -0
- package/src/config/enhanced-ragrc-schema.js +632 -0
- package/src/config/load-config.js +54 -0
- package/src/config/load-plugin-config.js +69 -0
- package/src/config/normalize-config.js +105 -0
- package/src/config/schema-validator.js +315 -0
- package/src/config/validate-plugin-schema.js +35 -0
- package/src/config/validate-schema.js +113 -0
- package/src/contracts/plugin-contracts.js +33 -0
- package/src/core/create-pipeline.js +99 -0
- package/src/core/dependency-injection.js +242 -0
- package/src/core/observability/event-logger.js +241 -0
- package/src/core/observability/instrumented-pipeline.js +609 -0
- package/src/core/observability/metrics.js +497 -0
- package/src/core/observability/tracing.js +781 -0
- package/src/core/performance/benchmark.js +516 -0
- package/src/core/performance/parallel-processor.js +240 -0
- package/src/core/performance/streaming-safeguards.js +611 -0
- package/src/core/pipeline-factory.js +60 -0
- package/src/core/plugin-contracts.js +89 -0
- package/src/core/plugin-marketplace/plugin-metadata.js +485 -0
- package/src/core/plugin-marketplace/plugin-publisher.js +574 -0
- package/src/core/plugin-marketplace/plugin-registry-format.js +419 -0
- package/src/core/plugin-marketplace/version-resolver.js +61 -0
- package/src/core/plugin-registry.js +265 -0
- package/src/dag/dag-engine.js +954 -0
- package/src/dag/dag-node.js +76 -0
- package/src/dag/enhanced-dag-engine.js +822 -0
- package/src/dx/index.js +40 -0
- package/src/dx/integration-templates.js +665 -0
- package/src/dx/performance-profiler.js +968 -0
- package/src/dx/realtime-debugger.js +755 -0
- package/src/dx/visual-pipeline-builder.js +578 -0
- package/src/ecosystem/plugin-analytics-dashboard.js +778 -0
- package/src/ecosystem/plugin-certification.js +580 -0
- package/src/ecosystem/plugin-hub.js +821 -0
- package/src/enterprise/audit-logging.js +967 -0
- package/src/enterprise/data-governance.js +408 -0
- package/src/enterprise/multi-tenancy.js +496 -0
- package/src/enterprise/sso-integration.js +903 -0
- package/src/evaluate/evaluator.js +52 -0
- package/src/evaluate/scoring.js +91 -0
- package/src/index.js +67 -0
- package/src/ingest.js +59 -0
- package/src/loader/csv-loader.js +38 -0
- package/src/loader/directory-loader.js +48 -0
- package/src/loader/html-loader.js +62 -0
- package/src/loader/markdown-loader.js +67 -0
- package/src/mocks/openai-embedder.js +119 -0
- package/src/mocks/openai-llm.js +77 -0
- package/src/mocks/pdf-loader.js +21 -0
- package/src/mocks/pinecone-retriever.js +106 -0
- package/src/observability/audit-logger.js +758 -0
- package/src/observability/dashboard-config.json +675 -0
- package/src/observability/event-logger.js +310 -0
- package/src/observability/metrics.js +691 -0
- package/src/observability/security-monitor.js +830 -0
- package/src/observability/slo-monitor.js +319 -0
- package/src/observability/telemetry.js +686 -0
- package/src/pipeline/pipeline.js +18 -0
- package/src/plugins/sample-embedder.js +53 -0
- package/src/plugins/sample-llm.js +30 -0
- package/src/plugins/sample-loader.js +24 -0
- package/src/plugins/sample-retriever.js +84 -0
- package/src/query.js +117 -0
- package/src/reranker/llm-reranker.js +85 -0
- package/src/scripts/script-utilities.js +36 -0
- package/src/security/plugin-sandbox.js +586 -0
- package/src/security/plugin-signature-verifier.js +414 -0
- package/src/security/validator.js +36 -0
- package/src/utils/ci/diagnostic-reporter.js +57 -0
- package/src/utils/logger.js +211 -0
- package/src/utils/performance-monitor.js +561 -0
- package/src/utils/plugin-scaffolder.js +447 -0
- package/src/utils/retry-policy.js +443 -0
- package/src/utils/retry.js +107 -0
- package/src/utils/safe-expression-evaluator.js +579 -0
- package/src/utils/structured-logger.js +361 -0
- package/src/utils/validate-plugin-contract.js +42 -0
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const {
|
|
6
|
+
createPluginVerifier,
|
|
7
|
+
} = require('../security/plugin-signature-verifier.js');
|
|
8
|
+
|
|
9
|
+
class PluginRegistry {
|
|
10
|
+
constructor(options = {}) {
|
|
11
|
+
this._plugins = new Map();
|
|
12
|
+
this._validTypes = new Set([
|
|
13
|
+
'loader',
|
|
14
|
+
'embedder',
|
|
15
|
+
'retriever',
|
|
16
|
+
'reranker',
|
|
17
|
+
'llm',
|
|
18
|
+
'evaluator',
|
|
19
|
+
]);
|
|
20
|
+
this._contracts = new Map();
|
|
21
|
+
|
|
22
|
+
// Environment-aware signature verification defaults
|
|
23
|
+
const isDevelopment = process.env.NODE_ENV === 'development';
|
|
24
|
+
const isProduction = process.env.NODE_ENV === 'production';
|
|
25
|
+
|
|
26
|
+
this._signatureVerifier = createPluginVerifier({
|
|
27
|
+
enabled:
|
|
28
|
+
options.verifySignatures !== undefined
|
|
29
|
+
? options.verifySignatures
|
|
30
|
+
: isProduction, // Enabled by default in production, disabled in development
|
|
31
|
+
failClosed:
|
|
32
|
+
options.failClosed !== undefined ? options.failClosed : isProduction, // Fail closed in production, fail open in development
|
|
33
|
+
trustedKeysPath: options.trustedKeysPath,
|
|
34
|
+
});
|
|
35
|
+
this._loadContracts();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
_loadContracts() {
|
|
39
|
+
const contractsDir = path.join(__dirname, '../../contracts');
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
const contractFiles = {
|
|
43
|
+
loader: 'loader-contract.json',
|
|
44
|
+
embedder: 'embedder-contract.json',
|
|
45
|
+
retriever: 'retriever-contract.json',
|
|
46
|
+
llm: 'llm-contract.json',
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
for (const [type, filename] of Object.entries(contractFiles)) {
|
|
50
|
+
try {
|
|
51
|
+
const contractPath = path.join(contractsDir, filename);
|
|
52
|
+
if (fs.existsSync(contractPath)) {
|
|
53
|
+
const contractContent = fs.readFileSync(contractPath, 'utf8');
|
|
54
|
+
const contract = JSON.parse(contractContent);
|
|
55
|
+
this._contracts.set(type, contract);
|
|
56
|
+
}
|
|
57
|
+
} catch (error) {
|
|
58
|
+
// Contract loading is optional - don't fail if missing
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
} catch (error) {
|
|
62
|
+
// Contracts directory may not exist in all environments
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
_validatePluginContract(category, impl) {
|
|
67
|
+
const contract = this._contracts.get(category);
|
|
68
|
+
if (!contract) {
|
|
69
|
+
return; // No contract to validate against
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Check if plugin has metadata
|
|
73
|
+
if (!impl.metadata) {
|
|
74
|
+
throw new Error('Plugin missing metadata property');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (!impl.metadata.name || typeof impl.metadata.name !== 'string') {
|
|
78
|
+
throw new Error('Plugin metadata must have a name property');
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (!impl.metadata.version || typeof impl.metadata.version !== 'string') {
|
|
82
|
+
throw new Error('Plugin metadata must have a version property');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (!impl.metadata.type || impl.metadata.type !== category) {
|
|
86
|
+
throw new Error(`Plugin metadata type must be '${category}'`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Validate required methods based on contract
|
|
90
|
+
const requiredMethods = this._getRequiredMethodsFromContract(contract);
|
|
91
|
+
|
|
92
|
+
for (const methodName of requiredMethods) {
|
|
93
|
+
if (!impl[methodName]) {
|
|
94
|
+
throw new Error(`Plugin missing required method: ${methodName}`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (typeof impl[methodName] !== 'function') {
|
|
98
|
+
throw new Error(`Plugin method '${methodName}' must be a function`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
_getRequiredMethodsFromContract(contract) {
|
|
104
|
+
const methods = [];
|
|
105
|
+
|
|
106
|
+
if (contract.required && Array.isArray(contract.required)) {
|
|
107
|
+
methods.push(...contract.required);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (contract.properties) {
|
|
111
|
+
for (const [methodName, methodSpec] of Object.entries(
|
|
112
|
+
contract.properties,
|
|
113
|
+
)) {
|
|
114
|
+
if (methodSpec.type === 'function') {
|
|
115
|
+
methods.push(methodName);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return methods;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async register(category, name, impl, manifest = null) {
|
|
124
|
+
if (!category || typeof category !== 'string') {
|
|
125
|
+
throw new Error('Category must be a non-empty string');
|
|
126
|
+
}
|
|
127
|
+
if (!name || typeof name !== 'string') {
|
|
128
|
+
throw new Error('Name must be a non-empty string');
|
|
129
|
+
}
|
|
130
|
+
if (impl == null) {
|
|
131
|
+
throw new Error('Implementation cannot be null or undefined');
|
|
132
|
+
}
|
|
133
|
+
if (!this._validTypes.has(category)) {
|
|
134
|
+
throw new Error(`Unknown plugin type: ${category}`);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Verify plugin signature if manifest provided
|
|
138
|
+
if (manifest) {
|
|
139
|
+
await this._verifyPluginSignature(manifest, name);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Validate plugin against contract
|
|
143
|
+
this._validatePluginContract(category, impl);
|
|
144
|
+
|
|
145
|
+
const key = `${category}:${name}`;
|
|
146
|
+
this._plugins.set(key, impl);
|
|
147
|
+
return this;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Verify plugin signature
|
|
152
|
+
* @param {object} manifest - Plugin manifest
|
|
153
|
+
* @param {string} pluginName - Plugin name
|
|
154
|
+
*/
|
|
155
|
+
async _verifyPluginSignature(manifest, pluginName) {
|
|
156
|
+
try {
|
|
157
|
+
// Extract signature information from manifest
|
|
158
|
+
if (!manifest.signature || !manifest.signerId) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`Plugin ${pluginName} missing required signature or signerId`,
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const verificationResult =
|
|
165
|
+
await this._signatureVerifier.verifyPluginSignature(
|
|
166
|
+
manifest,
|
|
167
|
+
manifest.signature,
|
|
168
|
+
manifest.signerId,
|
|
169
|
+
);
|
|
170
|
+
|
|
171
|
+
if (!verificationResult.verified) {
|
|
172
|
+
throw new Error(
|
|
173
|
+
`Plugin ${pluginName} signature verification failed: ${verificationResult.error}`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Emit audit entry for successful verification
|
|
178
|
+
this._emitAuditEntry('plugin_verified', {
|
|
179
|
+
pluginName,
|
|
180
|
+
signerId: manifest.signerId,
|
|
181
|
+
version: manifest.version,
|
|
182
|
+
verified: true,
|
|
183
|
+
timestamp: new Date().toISOString(),
|
|
184
|
+
});
|
|
185
|
+
} catch (error) {
|
|
186
|
+
// Emit audit entry for failed verification
|
|
187
|
+
this._emitAuditEntry('plugin_verification_failed', {
|
|
188
|
+
pluginName,
|
|
189
|
+
signerId: manifest.signerId || 'unknown',
|
|
190
|
+
version: manifest.version || 'unknown',
|
|
191
|
+
verified: false,
|
|
192
|
+
error: error.message,
|
|
193
|
+
timestamp: new Date().toISOString(),
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Emit audit entry
|
|
202
|
+
* @param {string} action - Audit action
|
|
203
|
+
* @param {object} details - Audit details
|
|
204
|
+
*/
|
|
205
|
+
_emitAuditEntry(action, details) {
|
|
206
|
+
const auditEntry = {
|
|
207
|
+
timestamp: new Date().toISOString(),
|
|
208
|
+
action,
|
|
209
|
+
component: 'plugin_registry',
|
|
210
|
+
...details,
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
// In production, this should go to a secure audit log
|
|
214
|
+
if (process.env.NODE_ENV === 'production') {
|
|
215
|
+
console.warn('[AUDIT]', JSON.stringify(auditEntry)); // eslint-disable-line no-console
|
|
216
|
+
} else {
|
|
217
|
+
console.log('[AUDIT]', auditEntry); // eslint-disable-line no-console
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
get(category, name) {
|
|
222
|
+
const key = `${category}:${name}`;
|
|
223
|
+
const impl = this._plugins.get(key);
|
|
224
|
+
if (impl === undefined) {
|
|
225
|
+
throw new Error(`Plugin not found: ${category}/${name}`);
|
|
226
|
+
}
|
|
227
|
+
return impl;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
resolve(category, name) {
|
|
231
|
+
return this.get(category, name);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
list(category) {
|
|
235
|
+
if (!this._validTypes.has(category)) {
|
|
236
|
+
throw new Error(`Unknown plugin type: ${category}`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const names = [];
|
|
240
|
+
const prefix = `${category}:`;
|
|
241
|
+
|
|
242
|
+
for (const key of this._plugins.keys()) {
|
|
243
|
+
if (key.startsWith(prefix)) {
|
|
244
|
+
names.push(key.substring(prefix.length));
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return names;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
clear() {
|
|
252
|
+
this._plugins.clear();
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Create singleton instance with default security settings
|
|
257
|
+
const registry = new PluginRegistry({
|
|
258
|
+
verifySignatures: process.env.NODE_ENV === 'production',
|
|
259
|
+
failClosed: process.env.NODE_ENV === 'production',
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
// CJS+ESM interop pattern
|
|
263
|
+
module.exports = registry;
|
|
264
|
+
module.exports.PluginRegistry = PluginRegistry;
|
|
265
|
+
module.exports.default = module.exports;
|