@devilsdev/rag-pipeline-utils 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +260 -19
- package/contracts/contract-schema.json +189 -0
- package/dist/index.cjs +19 -0
- package/dist/index.mjs +15 -1
- package/package.json +3 -2
- package/src/cli/commands/wizard.js +627 -0
- package/src/cli/input-sanitization-middleware.js +356 -0
- package/src/cli/templates/development.template.json +64 -0
- package/src/cli/templates/minimal.template.json +33 -0
- package/src/cli/templates/production.template.json +104 -0
- package/src/cli/templates/testing.template.json +45 -0
- package/src/config/connection-pool-config.json +259 -0
- package/src/config/memory-optimization.js +414 -0
- package/src/config/redaction-patterns.json +150 -0
- package/src/config/sanitization-rules.json +323 -0
- package/src/core/plugin-registry.js +266 -20
- package/src/dag/core/error-context.js +391 -0
- package/src/dag/core/execution-scheduler.js +350 -0
- package/src/dag/core/topology-validation.js +415 -0
- package/src/dag/dag-engine.js +26 -424
- package/src/dev/README.md +126 -0
- package/src/dev/dev-server.js +356 -0
- package/src/dev/examples/hot-reload-usage.js +355 -0
- package/src/dev/hot-reload.js +807 -0
- package/src/enterprise/sso-integration.js +67 -25
- package/src/index.js +19 -0
- package/src/observability/event-logger.js +62 -2
- package/src/plugins/optimized-openai-embedder.js +169 -0
- package/src/security/jwt-config.js +431 -0
- package/src/security/jwt-validator.js +701 -0
- package/src/utils/api-client-pool-integration.js +436 -0
- package/src/utils/batch-processor.js +710 -0
- package/src/utils/connection-pool.js +605 -0
- package/src/utils/contract-validator.js +460 -0
- package/src/utils/error-formatter.js +603 -0
- package/src/utils/examples/error-formatter-usage.js +351 -0
- package/src/utils/input-sanitizer.js +967 -0
- package/src/utils/memory-monitor.js +800 -0
- package/src/utils/secure-logger-config.js +352 -0
- package/src/utils/secure-logger.js +576 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const fs = require('fs');
|
|
4
4
|
const path = require('path');
|
|
5
|
+
const Ajv = require('ajv');
|
|
5
6
|
const {
|
|
6
7
|
createPluginVerifier,
|
|
7
8
|
} = require('../security/plugin-signature-verifier.js');
|
|
@@ -18,6 +19,40 @@ class PluginRegistry {
|
|
|
18
19
|
'evaluator',
|
|
19
20
|
]);
|
|
20
21
|
this._contracts = new Map();
|
|
22
|
+
this._contractWarningsShown = new Set(); // Track warnings to show only once per type
|
|
23
|
+
this._disableContractWarnings = options.disableContractWarnings || false;
|
|
24
|
+
this._validateContractSchema =
|
|
25
|
+
options.validateContractSchema !== undefined
|
|
26
|
+
? options.validateContractSchema
|
|
27
|
+
: true; // Enable schema validation by default
|
|
28
|
+
|
|
29
|
+
// Initialize Ajv for contract schema validation
|
|
30
|
+
this._ajv = new Ajv({
|
|
31
|
+
allErrors: true,
|
|
32
|
+
verbose: true,
|
|
33
|
+
strict: false,
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Load contract schema
|
|
37
|
+
this._contractSchema = this._loadContractSchema();
|
|
38
|
+
|
|
39
|
+
// Compile schema validator once
|
|
40
|
+
this._contractSchemaValidator = null;
|
|
41
|
+
if (this._contractSchema) {
|
|
42
|
+
try {
|
|
43
|
+
this._contractSchemaValidator = this._ajv.compile(this._contractSchema);
|
|
44
|
+
} catch (error) {
|
|
45
|
+
if (
|
|
46
|
+
process.env.NODE_ENV !== 'production' &&
|
|
47
|
+
!this._disableContractWarnings
|
|
48
|
+
) {
|
|
49
|
+
console.warn(
|
|
50
|
+
`[PLUGIN_REGISTRY] Warning: Failed to compile contract schema: ${error.message}`,
|
|
51
|
+
); // eslint-disable-line no-console
|
|
52
|
+
}
|
|
53
|
+
this._contractSchema = null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
21
56
|
|
|
22
57
|
// Environment-aware signature verification defaults
|
|
23
58
|
const isDevelopment = process.env.NODE_ENV === 'development';
|
|
@@ -35,9 +70,192 @@ class PluginRegistry {
|
|
|
35
70
|
this._loadContracts();
|
|
36
71
|
}
|
|
37
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Load contract schema for validation
|
|
75
|
+
*
|
|
76
|
+
* @private
|
|
77
|
+
* @returns {Object|null} Contract schema or null if not found
|
|
78
|
+
*
|
|
79
|
+
* @since 2.2.4
|
|
80
|
+
*/
|
|
81
|
+
_loadContractSchema() {
|
|
82
|
+
try {
|
|
83
|
+
const schemaPath = path.join(
|
|
84
|
+
__dirname,
|
|
85
|
+
'../../contracts/contract-schema.json',
|
|
86
|
+
);
|
|
87
|
+
if (fs.existsSync(schemaPath)) {
|
|
88
|
+
const schemaContent = fs.readFileSync(schemaPath, 'utf8');
|
|
89
|
+
return JSON.parse(schemaContent);
|
|
90
|
+
}
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (
|
|
93
|
+
process.env.NODE_ENV !== 'production' &&
|
|
94
|
+
!this._disableContractWarnings
|
|
95
|
+
) {
|
|
96
|
+
console.warn(
|
|
97
|
+
`[PLUGIN_REGISTRY] Warning: Failed to load contract schema: ${error.message}`,
|
|
98
|
+
); // eslint-disable-line no-console
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Validate contract against schema
|
|
106
|
+
*
|
|
107
|
+
* @private
|
|
108
|
+
* @param {Object} contract - Contract to validate
|
|
109
|
+
* @param {string} type - Contract type
|
|
110
|
+
* @returns {Object} Validation result { valid: boolean, errors: Array }
|
|
111
|
+
*
|
|
112
|
+
* @since 2.2.4
|
|
113
|
+
*/
|
|
114
|
+
_validateContractAgainstSchema(contract, type) {
|
|
115
|
+
if (!this._validateContractSchema || !this._contractSchemaValidator) {
|
|
116
|
+
return { valid: true, errors: [] };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
try {
|
|
120
|
+
const valid = this._contractSchemaValidator(contract);
|
|
121
|
+
|
|
122
|
+
if (!valid) {
|
|
123
|
+
return {
|
|
124
|
+
valid: false,
|
|
125
|
+
errors: this._contractSchemaValidator.errors || [],
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return { valid: true, errors: [] };
|
|
130
|
+
} catch (error) {
|
|
131
|
+
return {
|
|
132
|
+
valid: false,
|
|
133
|
+
errors: [
|
|
134
|
+
{
|
|
135
|
+
message: `Schema validation error: ${error.message}`,
|
|
136
|
+
},
|
|
137
|
+
],
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Format schema validation errors for display
|
|
144
|
+
*
|
|
145
|
+
* @private
|
|
146
|
+
* @param {Array} errors - Ajv validation errors
|
|
147
|
+
* @returns {string} Formatted error message
|
|
148
|
+
*
|
|
149
|
+
* @since 2.2.4
|
|
150
|
+
*/
|
|
151
|
+
_formatSchemaErrors(errors) {
|
|
152
|
+
if (!errors || errors.length === 0) {
|
|
153
|
+
return 'Unknown schema validation error';
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const messages = errors.map((err) => {
|
|
157
|
+
const path = err.instancePath || err.dataPath || '/';
|
|
158
|
+
const message = err.message || 'validation failed';
|
|
159
|
+
|
|
160
|
+
if (err.params) {
|
|
161
|
+
const params = Object.entries(err.params)
|
|
162
|
+
.map(([key, value]) => `${key}=${JSON.stringify(value)}`)
|
|
163
|
+
.join(', ');
|
|
164
|
+
return ` - ${path}: ${message} (${params})`;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return ` - ${path}: ${message}`;
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
return messages.join('\n');
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Check if a contract exists for the given plugin type
|
|
175
|
+
*
|
|
176
|
+
* @param {string} type - Plugin type (loader, embedder, etc.)
|
|
177
|
+
* @returns {boolean} True if contract exists, false otherwise
|
|
178
|
+
*
|
|
179
|
+
* @example
|
|
180
|
+
* const hasContract = registry.checkContractExists('loader');
|
|
181
|
+
* if (!hasContract) {
|
|
182
|
+
* console.warn('No contract for loader plugins');
|
|
183
|
+
* }
|
|
184
|
+
*
|
|
185
|
+
* @since 2.2.4
|
|
186
|
+
*/
|
|
187
|
+
checkContractExists(type) {
|
|
188
|
+
return this._contracts.has(type);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Log one-time warning for missing contract
|
|
193
|
+
*
|
|
194
|
+
* Logs a helpful warning message about the benefits of contracts
|
|
195
|
+
* and how to add them. Only logs once per contract type and only
|
|
196
|
+
* in non-production environments.
|
|
197
|
+
*
|
|
198
|
+
* @private
|
|
199
|
+
* @param {string} type - Plugin type
|
|
200
|
+
* @param {string} context - Context where warning is generated (e.g., 'load', 'register')
|
|
201
|
+
*
|
|
202
|
+
* @since 2.2.4
|
|
203
|
+
*/
|
|
204
|
+
_warnMissingContract(type, context = 'load') {
|
|
205
|
+
// Skip if warnings disabled or in production
|
|
206
|
+
if (
|
|
207
|
+
this._disableContractWarnings ||
|
|
208
|
+
process.env.NODE_ENV === 'production'
|
|
209
|
+
) {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Only warn once per type
|
|
214
|
+
const warningKey = `${type}:${context}`;
|
|
215
|
+
if (this._contractWarningsShown.has(warningKey)) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
this._contractWarningsShown.add(warningKey);
|
|
220
|
+
|
|
221
|
+
console.warn(
|
|
222
|
+
`
|
|
223
|
+
╔════════════════════════════════════════════════════════════════════════════╗
|
|
224
|
+
║ [PLUGIN_REGISTRY] Missing Contract for '${type}' plugins ║
|
|
225
|
+
╠════════════════════════════════════════════════════════════════════════════╣
|
|
226
|
+
║ A contract specification was not found for the '${type}' plugin type. ║
|
|
227
|
+
║ ║
|
|
228
|
+
║ Benefits of using contracts: ║
|
|
229
|
+
║ • Ensures plugin compatibility across versions ║
|
|
230
|
+
║ • Validates required methods and properties ║
|
|
231
|
+
║ • Provides clear interface documentation ║
|
|
232
|
+
║ • Catches errors early in development ║
|
|
233
|
+
║ • Enables automated testing and validation ║
|
|
234
|
+
║ ║
|
|
235
|
+
║ To add a contract: ║
|
|
236
|
+
║ 1. Create contracts/${type}-contract.json ║
|
|
237
|
+
║ 2. Define required methods and their signatures ║
|
|
238
|
+
║ 3. Run contract validation tests ║
|
|
239
|
+
║ ║
|
|
240
|
+
║ Example contract structure: ║
|
|
241
|
+
║ { ║
|
|
242
|
+
║ "type": "${type}", ║
|
|
243
|
+
║ "version": "1.0.0", ║
|
|
244
|
+
║ "required": ["methodName"], ║
|
|
245
|
+
║ "properties": { /* method definitions */ } ║
|
|
246
|
+
║ } ║
|
|
247
|
+
║ ║
|
|
248
|
+
║ See: docs/error-context-guide.md and __tests__/contracts/README.md ║
|
|
249
|
+
║ ║
|
|
250
|
+
║ To disable these warnings: ║
|
|
251
|
+
║ new PluginRegistry({ disableContractWarnings: true }) ║
|
|
252
|
+
╚════════════════════════════════════════════════════════════════════════════╝
|
|
253
|
+
`.trim(),
|
|
254
|
+
); // eslint-disable-line no-console
|
|
255
|
+
}
|
|
256
|
+
|
|
38
257
|
_loadContracts() {
|
|
39
258
|
const contractsDir = path.join(__dirname, '../../contracts');
|
|
40
|
-
const isDevelopment = process.env.NODE_ENV === 'development';
|
|
41
259
|
|
|
42
260
|
try {
|
|
43
261
|
const contractFiles = {
|
|
@@ -54,19 +272,46 @@ class PluginRegistry {
|
|
|
54
272
|
if (fs.existsSync(contractPath)) {
|
|
55
273
|
const contractContent = fs.readFileSync(contractPath, 'utf8');
|
|
56
274
|
const contract = JSON.parse(contractContent);
|
|
275
|
+
|
|
276
|
+
// Validate contract against schema
|
|
277
|
+
const validation = this._validateContractAgainstSchema(
|
|
278
|
+
contract,
|
|
279
|
+
type,
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
if (!validation.valid) {
|
|
283
|
+
const errorMessage = this._formatSchemaErrors(validation.errors);
|
|
284
|
+
const error = new Error(
|
|
285
|
+
`Contract schema validation failed for '${type}':\n${errorMessage}`,
|
|
286
|
+
);
|
|
287
|
+
error.validationErrors = validation.errors;
|
|
288
|
+
error.contractType = type;
|
|
289
|
+
|
|
290
|
+
if (process.env.NODE_ENV === 'production') {
|
|
291
|
+
// In production, log error but continue (fail open)
|
|
292
|
+
console.error(`[PLUGIN_REGISTRY] ERROR: ${error.message}`); // eslint-disable-line no-console
|
|
293
|
+
} else {
|
|
294
|
+
// In development, throw to fail fast
|
|
295
|
+
throw error;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
57
299
|
this._contracts.set(type, contract);
|
|
58
|
-
} else
|
|
59
|
-
//
|
|
60
|
-
|
|
61
|
-
`[PLUGIN_REGISTRY] Warning: Contract file missing for '${type}' plugin type: ${contractPath}`,
|
|
62
|
-
); // eslint-disable-line no-console
|
|
63
|
-
console.warn(
|
|
64
|
-
`[PLUGIN_REGISTRY] This may indicate test-only drift. Contract validation will be skipped for '${type}' plugins.`,
|
|
65
|
-
); // eslint-disable-line no-console
|
|
300
|
+
} else {
|
|
301
|
+
// Use new warning method for missing contracts
|
|
302
|
+
this._warnMissingContract(type, 'load');
|
|
66
303
|
}
|
|
67
304
|
} catch (error) {
|
|
305
|
+
// Re-throw validation errors in development mode
|
|
306
|
+
if (error.validationErrors && process.env.NODE_ENV !== 'production') {
|
|
307
|
+
throw error;
|
|
308
|
+
}
|
|
309
|
+
|
|
68
310
|
// Contract loading is optional - don't fail if missing
|
|
69
|
-
if (
|
|
311
|
+
if (
|
|
312
|
+
process.env.NODE_ENV !== 'production' &&
|
|
313
|
+
!this._disableContractWarnings
|
|
314
|
+
) {
|
|
70
315
|
console.warn(
|
|
71
316
|
`[PLUGIN_REGISTRY] Warning: Failed to load contract for '${type}': ${error.message}`,
|
|
72
317
|
); // eslint-disable-line no-console
|
|
@@ -74,8 +319,16 @@ class PluginRegistry {
|
|
|
74
319
|
}
|
|
75
320
|
}
|
|
76
321
|
} catch (error) {
|
|
322
|
+
// Re-throw validation errors in development mode
|
|
323
|
+
if (error.validationErrors && process.env.NODE_ENV !== 'production') {
|
|
324
|
+
throw error;
|
|
325
|
+
}
|
|
326
|
+
|
|
77
327
|
// Contracts directory may not exist in all environments
|
|
78
|
-
if (
|
|
328
|
+
if (
|
|
329
|
+
process.env.NODE_ENV !== 'production' &&
|
|
330
|
+
!this._disableContractWarnings
|
|
331
|
+
) {
|
|
79
332
|
console.warn(
|
|
80
333
|
`[PLUGIN_REGISTRY] Warning: Contracts directory not accessible: ${error.message}`,
|
|
81
334
|
); // eslint-disable-line no-console
|
|
@@ -86,15 +339,8 @@ class PluginRegistry {
|
|
|
86
339
|
_validatePluginContract(category, impl) {
|
|
87
340
|
const contract = this._contracts.get(category);
|
|
88
341
|
if (!contract) {
|
|
89
|
-
//
|
|
90
|
-
|
|
91
|
-
console.warn(
|
|
92
|
-
`[PLUGIN_REGISTRY] Warning: No contract available for plugin category '${category}'. Skipping contract validation.`,
|
|
93
|
-
); // eslint-disable-line no-console
|
|
94
|
-
console.warn(
|
|
95
|
-
`[PLUGIN_REGISTRY] This may indicate test-only drift. Ensure contracts/${category}-contract.json exists.`,
|
|
96
|
-
); // eslint-disable-line no-console
|
|
97
|
-
}
|
|
342
|
+
// Use new warning method when registering a plugin without a contract
|
|
343
|
+
this._warnMissingContract(category, 'register');
|
|
98
344
|
return; // No contract to validate against
|
|
99
345
|
}
|
|
100
346
|
|