@devilsdev/rag-pipeline-utils 2.2.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +244 -11
  2. package/contracts/contract-schema.json +189 -0
  3. package/dist/index.cjs +19 -0
  4. package/dist/index.mjs +15 -1
  5. package/package.json +3 -2
  6. package/src/cli/commands/wizard.js +627 -0
  7. package/src/cli/input-sanitization-middleware.js +356 -0
  8. package/src/cli/templates/development.template.json +64 -0
  9. package/src/cli/templates/minimal.template.json +33 -0
  10. package/src/cli/templates/production.template.json +104 -0
  11. package/src/cli/templates/testing.template.json +45 -0
  12. package/src/config/connection-pool-config.json +259 -0
  13. package/src/config/memory-optimization.js +414 -0
  14. package/src/config/redaction-patterns.json +150 -0
  15. package/src/config/sanitization-rules.json +323 -0
  16. package/src/core/plugin-registry.js +266 -20
  17. package/src/dag/core/error-context.js +391 -0
  18. package/src/dag/core/execution-scheduler.js +350 -0
  19. package/src/dag/core/topology-validation.js +415 -0
  20. package/src/dag/dag-engine.js +26 -424
  21. package/src/dev/README.md +126 -0
  22. package/src/dev/dev-server.js +356 -0
  23. package/src/dev/examples/hot-reload-usage.js +355 -0
  24. package/src/dev/hot-reload.js +807 -0
  25. package/src/enterprise/sso-integration.js +67 -25
  26. package/src/index.js +19 -0
  27. package/src/observability/event-logger.js +62 -2
  28. package/src/plugins/optimized-openai-embedder.js +169 -0
  29. package/src/security/jwt-config.js +431 -0
  30. package/src/security/jwt-validator.js +701 -0
  31. package/src/utils/api-client-pool-integration.js +436 -0
  32. package/src/utils/batch-processor.js +710 -0
  33. package/src/utils/connection-pool.js +605 -0
  34. package/src/utils/contract-validator.js +460 -0
  35. package/src/utils/error-formatter.js +603 -0
  36. package/src/utils/examples/error-formatter-usage.js +351 -0
  37. package/src/utils/input-sanitizer.js +967 -0
  38. package/src/utils/memory-monitor.js +800 -0
  39. package/src/utils/secure-logger-config.js +352 -0
  40. package/src/utils/secure-logger.js +576 -0
@@ -2,6 +2,7 @@
2
2
 
3
3
  const fs = require('fs');
4
4
  const path = require('path');
5
+ const Ajv = require('ajv');
5
6
  const {
6
7
  createPluginVerifier,
7
8
  } = require('../security/plugin-signature-verifier.js');
@@ -18,6 +19,40 @@ class PluginRegistry {
18
19
  'evaluator',
19
20
  ]);
20
21
  this._contracts = new Map();
22
+ this._contractWarningsShown = new Set(); // Track warnings to show only once per type
23
+ this._disableContractWarnings = options.disableContractWarnings || false;
24
+ this._validateContractSchema =
25
+ options.validateContractSchema !== undefined
26
+ ? options.validateContractSchema
27
+ : true; // Enable schema validation by default
28
+
29
+ // Initialize Ajv for contract schema validation
30
+ this._ajv = new Ajv({
31
+ allErrors: true,
32
+ verbose: true,
33
+ strict: false,
34
+ });
35
+
36
+ // Load contract schema
37
+ this._contractSchema = this._loadContractSchema();
38
+
39
+ // Compile schema validator once
40
+ this._contractSchemaValidator = null;
41
+ if (this._contractSchema) {
42
+ try {
43
+ this._contractSchemaValidator = this._ajv.compile(this._contractSchema);
44
+ } catch (error) {
45
+ if (
46
+ process.env.NODE_ENV !== 'production' &&
47
+ !this._disableContractWarnings
48
+ ) {
49
+ console.warn(
50
+ `[PLUGIN_REGISTRY] Warning: Failed to compile contract schema: ${error.message}`,
51
+ ); // eslint-disable-line no-console
52
+ }
53
+ this._contractSchema = null;
54
+ }
55
+ }
21
56
 
22
57
  // Environment-aware signature verification defaults
23
58
  const isDevelopment = process.env.NODE_ENV === 'development';
@@ -35,9 +70,192 @@ class PluginRegistry {
35
70
  this._loadContracts();
36
71
  }
37
72
 
73
+ /**
74
+ * Load contract schema for validation
75
+ *
76
+ * @private
77
+ * @returns {Object|null} Contract schema or null if not found
78
+ *
79
+ * @since 2.2.4
80
+ */
81
+ _loadContractSchema() {
82
+ try {
83
+ const schemaPath = path.join(
84
+ __dirname,
85
+ '../../contracts/contract-schema.json',
86
+ );
87
+ if (fs.existsSync(schemaPath)) {
88
+ const schemaContent = fs.readFileSync(schemaPath, 'utf8');
89
+ return JSON.parse(schemaContent);
90
+ }
91
+ } catch (error) {
92
+ if (
93
+ process.env.NODE_ENV !== 'production' &&
94
+ !this._disableContractWarnings
95
+ ) {
96
+ console.warn(
97
+ `[PLUGIN_REGISTRY] Warning: Failed to load contract schema: ${error.message}`,
98
+ ); // eslint-disable-line no-console
99
+ }
100
+ }
101
+ return null;
102
+ }
103
+
104
+ /**
105
+ * Validate contract against schema
106
+ *
107
+ * @private
108
+ * @param {Object} contract - Contract to validate
109
+ * @param {string} type - Contract type
110
+ * @returns {Object} Validation result { valid: boolean, errors: Array }
111
+ *
112
+ * @since 2.2.4
113
+ */
114
+ _validateContractAgainstSchema(contract, type) {
115
+ if (!this._validateContractSchema || !this._contractSchemaValidator) {
116
+ return { valid: true, errors: [] };
117
+ }
118
+
119
+ try {
120
+ const valid = this._contractSchemaValidator(contract);
121
+
122
+ if (!valid) {
123
+ return {
124
+ valid: false,
125
+ errors: this._contractSchemaValidator.errors || [],
126
+ };
127
+ }
128
+
129
+ return { valid: true, errors: [] };
130
+ } catch (error) {
131
+ return {
132
+ valid: false,
133
+ errors: [
134
+ {
135
+ message: `Schema validation error: ${error.message}`,
136
+ },
137
+ ],
138
+ };
139
+ }
140
+ }
141
+
142
+ /**
143
+ * Format schema validation errors for display
144
+ *
145
+ * @private
146
+ * @param {Array} errors - Ajv validation errors
147
+ * @returns {string} Formatted error message
148
+ *
149
+ * @since 2.2.4
150
+ */
151
+ _formatSchemaErrors(errors) {
152
+ if (!errors || errors.length === 0) {
153
+ return 'Unknown schema validation error';
154
+ }
155
+
156
+ const messages = errors.map((err) => {
157
+ const path = err.instancePath || err.dataPath || '/';
158
+ const message = err.message || 'validation failed';
159
+
160
+ if (err.params) {
161
+ const params = Object.entries(err.params)
162
+ .map(([key, value]) => `${key}=${JSON.stringify(value)}`)
163
+ .join(', ');
164
+ return ` - ${path}: ${message} (${params})`;
165
+ }
166
+
167
+ return ` - ${path}: ${message}`;
168
+ });
169
+
170
+ return messages.join('\n');
171
+ }
172
+
173
+ /**
174
+ * Check if a contract exists for the given plugin type
175
+ *
176
+ * @param {string} type - Plugin type (loader, embedder, etc.)
177
+ * @returns {boolean} True if contract exists, false otherwise
178
+ *
179
+ * @example
180
+ * const hasContract = registry.checkContractExists('loader');
181
+ * if (!hasContract) {
182
+ * console.warn('No contract for loader plugins');
183
+ * }
184
+ *
185
+ * @since 2.2.4
186
+ */
187
+ checkContractExists(type) {
188
+ return this._contracts.has(type);
189
+ }
190
+
191
+ /**
192
+ * Log one-time warning for missing contract
193
+ *
194
+ * Logs a helpful warning message about the benefits of contracts
195
+ * and how to add them. Only logs once per contract type and only
196
+ * in non-production environments.
197
+ *
198
+ * @private
199
+ * @param {string} type - Plugin type
200
+ * @param {string} context - Context where warning is generated (e.g., 'load', 'register')
201
+ *
202
+ * @since 2.2.4
203
+ */
204
+ _warnMissingContract(type, context = 'load') {
205
+ // Skip if warnings disabled or in production
206
+ if (
207
+ this._disableContractWarnings ||
208
+ process.env.NODE_ENV === 'production'
209
+ ) {
210
+ return;
211
+ }
212
+
213
+ // Only warn once per type
214
+ const warningKey = `${type}:${context}`;
215
+ if (this._contractWarningsShown.has(warningKey)) {
216
+ return;
217
+ }
218
+
219
+ this._contractWarningsShown.add(warningKey);
220
+
221
+ console.warn(
222
+ `
223
+ ╔════════════════════════════════════════════════════════════════════════════╗
224
+ ║ [PLUGIN_REGISTRY] Missing Contract for '${type}' plugins ║
225
+ ╠════════════════════════════════════════════════════════════════════════════╣
226
+ ║ A contract specification was not found for the '${type}' plugin type. ║
227
+ ║ ║
228
+ ║ Benefits of using contracts: ║
229
+ ║ • Ensures plugin compatibility across versions ║
230
+ ║ • Validates required methods and properties ║
231
+ ║ • Provides clear interface documentation ║
232
+ ║ • Catches errors early in development ║
233
+ ║ • Enables automated testing and validation ║
234
+ ║ ║
235
+ ║ To add a contract: ║
236
+ ║ 1. Create contracts/${type}-contract.json ║
237
+ ║ 2. Define required methods and their signatures ║
238
+ ║ 3. Run contract validation tests ║
239
+ ║ ║
240
+ ║ Example contract structure: ║
241
+ ║ { ║
242
+ ║ "type": "${type}", ║
243
+ ║ "version": "1.0.0", ║
244
+ ║ "required": ["methodName"], ║
245
+ ║ "properties": { /* method definitions */ } ║
246
+ ║ } ║
247
+ ║ ║
248
+ ║ See: docs/error-context-guide.md and __tests__/contracts/README.md ║
249
+ ║ ║
250
+ ║ To disable these warnings: ║
251
+ ║ new PluginRegistry({ disableContractWarnings: true }) ║
252
+ ╚════════════════════════════════════════════════════════════════════════════╝
253
+ `.trim(),
254
+ ); // eslint-disable-line no-console
255
+ }
256
+
38
257
  _loadContracts() {
39
258
  const contractsDir = path.join(__dirname, '../../contracts');
40
- const isDevelopment = process.env.NODE_ENV === 'development';
41
259
 
42
260
  try {
43
261
  const contractFiles = {
@@ -54,19 +272,46 @@ class PluginRegistry {
54
272
  if (fs.existsSync(contractPath)) {
55
273
  const contractContent = fs.readFileSync(contractPath, 'utf8');
56
274
  const contract = JSON.parse(contractContent);
275
+
276
+ // Validate contract against schema
277
+ const validation = this._validateContractAgainstSchema(
278
+ contract,
279
+ type,
280
+ );
281
+
282
+ if (!validation.valid) {
283
+ const errorMessage = this._formatSchemaErrors(validation.errors);
284
+ const error = new Error(
285
+ `Contract schema validation failed for '${type}':\n${errorMessage}`,
286
+ );
287
+ error.validationErrors = validation.errors;
288
+ error.contractType = type;
289
+
290
+ if (process.env.NODE_ENV === 'production') {
291
+ // In production, log error but continue (fail open)
292
+ console.error(`[PLUGIN_REGISTRY] ERROR: ${error.message}`); // eslint-disable-line no-console
293
+ } else {
294
+ // In development, throw to fail fast
295
+ throw error;
296
+ }
297
+ }
298
+
57
299
  this._contracts.set(type, contract);
58
- } else if (isDevelopment) {
59
- // Dev mode warning: flag missing contract files
60
- console.warn(
61
- `[PLUGIN_REGISTRY] Warning: Contract file missing for '${type}' plugin type: ${contractPath}`,
62
- ); // eslint-disable-line no-console
63
- console.warn(
64
- `[PLUGIN_REGISTRY] This may indicate test-only drift. Contract validation will be skipped for '${type}' plugins.`,
65
- ); // eslint-disable-line no-console
300
+ } else {
301
+ // Use new warning method for missing contracts
302
+ this._warnMissingContract(type, 'load');
66
303
  }
67
304
  } catch (error) {
305
+ // Re-throw validation errors in development mode
306
+ if (error.validationErrors && process.env.NODE_ENV !== 'production') {
307
+ throw error;
308
+ }
309
+
68
310
  // Contract loading is optional - don't fail if missing
69
- if (isDevelopment) {
311
+ if (
312
+ process.env.NODE_ENV !== 'production' &&
313
+ !this._disableContractWarnings
314
+ ) {
70
315
  console.warn(
71
316
  `[PLUGIN_REGISTRY] Warning: Failed to load contract for '${type}': ${error.message}`,
72
317
  ); // eslint-disable-line no-console
@@ -74,8 +319,16 @@ class PluginRegistry {
74
319
  }
75
320
  }
76
321
  } catch (error) {
322
+ // Re-throw validation errors in development mode
323
+ if (error.validationErrors && process.env.NODE_ENV !== 'production') {
324
+ throw error;
325
+ }
326
+
77
327
  // Contracts directory may not exist in all environments
78
- if (isDevelopment) {
328
+ if (
329
+ process.env.NODE_ENV !== 'production' &&
330
+ !this._disableContractWarnings
331
+ ) {
79
332
  console.warn(
80
333
  `[PLUGIN_REGISTRY] Warning: Contracts directory not accessible: ${error.message}`,
81
334
  ); // eslint-disable-line no-console
@@ -86,15 +339,8 @@ class PluginRegistry {
86
339
  _validatePluginContract(category, impl) {
87
340
  const contract = this._contracts.get(category);
88
341
  if (!contract) {
89
- // Warn in dev mode when registering a plugin without a contract
90
- if (process.env.NODE_ENV === 'development') {
91
- console.warn(
92
- `[PLUGIN_REGISTRY] Warning: No contract available for plugin category '${category}'. Skipping contract validation.`,
93
- ); // eslint-disable-line no-console
94
- console.warn(
95
- `[PLUGIN_REGISTRY] This may indicate test-only drift. Ensure contracts/${category}-contract.json exists.`,
96
- ); // eslint-disable-line no-console
97
- }
342
+ // Use new warning method when registering a plugin without a contract
343
+ this._warnMissingContract(category, 'register');
98
344
  return; // No contract to validate against
99
345
  }
100
346