@devilsdev/rag-pipeline-utils 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +244 -11
- package/contracts/contract-schema.json +189 -0
- package/dist/index.cjs +19 -0
- package/dist/index.mjs +15 -1
- package/package.json +3 -2
- package/src/cli/commands/wizard.js +627 -0
- package/src/cli/input-sanitization-middleware.js +356 -0
- package/src/cli/templates/development.template.json +64 -0
- package/src/cli/templates/minimal.template.json +33 -0
- package/src/cli/templates/production.template.json +104 -0
- package/src/cli/templates/testing.template.json +45 -0
- package/src/config/connection-pool-config.json +259 -0
- package/src/config/memory-optimization.js +414 -0
- package/src/config/redaction-patterns.json +150 -0
- package/src/config/sanitization-rules.json +323 -0
- package/src/core/plugin-registry.js +266 -20
- package/src/dag/core/error-context.js +391 -0
- package/src/dag/core/execution-scheduler.js +350 -0
- package/src/dag/core/topology-validation.js +415 -0
- package/src/dag/dag-engine.js +26 -424
- package/src/dev/README.md +126 -0
- package/src/dev/dev-server.js +356 -0
- package/src/dev/examples/hot-reload-usage.js +355 -0
- package/src/dev/hot-reload.js +807 -0
- package/src/enterprise/sso-integration.js +67 -25
- package/src/index.js +19 -0
- package/src/observability/event-logger.js +62 -2
- package/src/plugins/optimized-openai-embedder.js +169 -0
- package/src/security/jwt-config.js +431 -0
- package/src/security/jwt-validator.js +701 -0
- package/src/utils/api-client-pool-integration.js +436 -0
- package/src/utils/batch-processor.js +710 -0
- package/src/utils/connection-pool.js +605 -0
- package/src/utils/contract-validator.js +460 -0
- package/src/utils/error-formatter.js +603 -0
- package/src/utils/examples/error-formatter-usage.js +351 -0
- package/src/utils/input-sanitizer.js +967 -0
- package/src/utils/memory-monitor.js +800 -0
- package/src/utils/secure-logger-config.js +352 -0
- package/src/utils/secure-logger.js +576 -0
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Error Formatter Usage Examples
|
|
5
|
+
*
|
|
6
|
+
* Demonstrates how to use the enhanced error formatter
|
|
7
|
+
* throughout the codebase
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const {
|
|
11
|
+
createError,
|
|
12
|
+
wrapError,
|
|
13
|
+
formatErrorMessage,
|
|
14
|
+
logError,
|
|
15
|
+
ERROR_CODES,
|
|
16
|
+
} = require('../error-formatter');
|
|
17
|
+
|
|
18
|
+
// ============================================================================
|
|
19
|
+
// Example 1: Basic Error Creation
|
|
20
|
+
// ============================================================================
|
|
21
|
+
|
|
22
|
+
function basicErrorExample() {
|
|
23
|
+
console.log('\n=== Example 1: Basic Error Creation ===\n');
|
|
24
|
+
|
|
25
|
+
try {
|
|
26
|
+
throw createError(ERROR_CODES.CONFIG_NOT_FOUND, {
|
|
27
|
+
path: '.ragrc.json',
|
|
28
|
+
});
|
|
29
|
+
} catch (error) {
|
|
30
|
+
console.log(formatErrorMessage(error));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// ============================================================================
|
|
35
|
+
// Example 2: Error with Rich Context
|
|
36
|
+
// ============================================================================
|
|
37
|
+
|
|
38
|
+
function richContextExample() {
|
|
39
|
+
console.log('\n=== Example 2: Error with Rich Context ===\n');
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
const input = { invalid: 'data' };
|
|
43
|
+
|
|
44
|
+
throw createError(ERROR_CODES.PLUGIN_EXECUTION_FAILED, {
|
|
45
|
+
plugin: 'embedder-openai',
|
|
46
|
+
reason: 'Invalid API key',
|
|
47
|
+
input: JSON.stringify(input),
|
|
48
|
+
details: 'Check your OPENAI_API_KEY environment variable',
|
|
49
|
+
});
|
|
50
|
+
} catch (error) {
|
|
51
|
+
console.log(
|
|
52
|
+
formatErrorMessage(error, {
|
|
53
|
+
includeContext: true,
|
|
54
|
+
includeSuggestions: true,
|
|
55
|
+
includeDocumentation: true,
|
|
56
|
+
}),
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ============================================================================
|
|
62
|
+
// Example 3: Wrapping Existing Errors
|
|
63
|
+
// ============================================================================
|
|
64
|
+
|
|
65
|
+
function wrapErrorExample() {
|
|
66
|
+
console.log('\n=== Example 3: Wrapping Existing Errors ===\n');
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
try {
|
|
70
|
+
// Simulate file read error
|
|
71
|
+
throw new Error('ENOENT: no such file or directory');
|
|
72
|
+
} catch (originalError) {
|
|
73
|
+
throw wrapError(originalError, ERROR_CODES.FS_FILE_NOT_FOUND, {
|
|
74
|
+
path: '/path/to/config.json',
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
} catch (error) {
|
|
78
|
+
console.log(
|
|
79
|
+
formatErrorMessage(error, {
|
|
80
|
+
includeStack: false,
|
|
81
|
+
}),
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ============================================================================
|
|
87
|
+
// Example 4: Configuration Errors
|
|
88
|
+
// ============================================================================
|
|
89
|
+
|
|
90
|
+
function configErrorExample() {
|
|
91
|
+
console.log('\n=== Example 4: Configuration Errors ===\n');
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
throw createError(ERROR_CODES.CONFIG_INVALID_TYPE, {
|
|
95
|
+
field: 'pipeline.stages',
|
|
96
|
+
expected: 'array',
|
|
97
|
+
actual: 'string',
|
|
98
|
+
currentValue: '"fetch,process,store"',
|
|
99
|
+
});
|
|
100
|
+
} catch (error) {
|
|
101
|
+
console.log(formatErrorMessage(error));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// ============================================================================
|
|
106
|
+
// Example 5: Plugin Contract Violation
|
|
107
|
+
// ============================================================================
|
|
108
|
+
|
|
109
|
+
function pluginContractExample() {
|
|
110
|
+
console.log('\n=== Example 5: Plugin Contract Violation ===\n');
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
throw createError(ERROR_CODES.PLUGIN_CONTRACT_VIOLATION, {
|
|
114
|
+
plugin: 'my-custom-plugin',
|
|
115
|
+
violation: 'Missing execute() method',
|
|
116
|
+
missingMethod: 'execute',
|
|
117
|
+
expectedType: 'function',
|
|
118
|
+
});
|
|
119
|
+
} catch (error) {
|
|
120
|
+
console.log(formatErrorMessage(error));
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ============================================================================
|
|
125
|
+
// Example 6: Pipeline Execution Error
|
|
126
|
+
// ============================================================================
|
|
127
|
+
|
|
128
|
+
function pipelineErrorExample() {
|
|
129
|
+
console.log('\n=== Example 6: Pipeline Execution Error ===\n');
|
|
130
|
+
|
|
131
|
+
try {
|
|
132
|
+
throw createError(ERROR_CODES.PIPELINE_STAGE_FAILED, {
|
|
133
|
+
stage: 'embedding',
|
|
134
|
+
reason: 'API rate limit exceeded',
|
|
135
|
+
input: JSON.stringify({ text: 'Sample text...' }).slice(0, 50),
|
|
136
|
+
});
|
|
137
|
+
} catch (error) {
|
|
138
|
+
// Log with appropriate severity
|
|
139
|
+
logError(error);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ============================================================================
|
|
144
|
+
// Example 7: Validation Error with Multiple Issues
|
|
145
|
+
// ============================================================================
|
|
146
|
+
|
|
147
|
+
function validationErrorExample() {
|
|
148
|
+
console.log('\n=== Example 7: Validation Error ===\n');
|
|
149
|
+
|
|
150
|
+
try {
|
|
151
|
+
const errors = [
|
|
152
|
+
{ field: 'metadata.name', message: 'Required field missing' },
|
|
153
|
+
{ field: 'pipeline.stages', message: 'Must be an array' },
|
|
154
|
+
];
|
|
155
|
+
|
|
156
|
+
throw createError(ERROR_CODES.VALIDATION_FAILED, {
|
|
157
|
+
reason: 'Configuration validation failed',
|
|
158
|
+
errors: JSON.stringify(errors, null, 2),
|
|
159
|
+
});
|
|
160
|
+
} catch (error) {
|
|
161
|
+
console.log(
|
|
162
|
+
formatErrorMessage(error, {
|
|
163
|
+
includeContext: true,
|
|
164
|
+
}),
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// ============================================================================
|
|
170
|
+
// Example 8: Hot Reload Error
|
|
171
|
+
// ============================================================================
|
|
172
|
+
|
|
173
|
+
function hotReloadErrorExample() {
|
|
174
|
+
console.log('\n=== Example 8: Hot Reload Error ===\n');
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
throw createError(ERROR_CODES.HOT_RELOAD_FAILED, {
|
|
178
|
+
path: './src/plugins/my-plugin.js',
|
|
179
|
+
reason: 'SyntaxError: Unexpected token',
|
|
180
|
+
watchPaths: './src/plugins, ./src/mocks',
|
|
181
|
+
});
|
|
182
|
+
} catch (error) {
|
|
183
|
+
console.log(formatErrorMessage(error));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ============================================================================
|
|
188
|
+
// Example 9: DAG Cycle Detection
|
|
189
|
+
// ============================================================================
|
|
190
|
+
|
|
191
|
+
function dagCycleExample() {
|
|
192
|
+
console.log('\n=== Example 9: DAG Cycle Detection ===\n');
|
|
193
|
+
|
|
194
|
+
try {
|
|
195
|
+
throw createError(ERROR_CODES.DAG_CYCLE_DETECTED, {
|
|
196
|
+
cycle: 'A -> B -> C -> A',
|
|
197
|
+
});
|
|
198
|
+
} catch (error) {
|
|
199
|
+
console.log(formatErrorMessage(error));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ============================================================================
|
|
204
|
+
// Example 10: Authentication Error
|
|
205
|
+
// ============================================================================
|
|
206
|
+
|
|
207
|
+
function authErrorExample() {
|
|
208
|
+
console.log('\n=== Example 10: Authentication Error ===\n');
|
|
209
|
+
|
|
210
|
+
try {
|
|
211
|
+
throw createError(ERROR_CODES.AUTH_TOKEN_EXPIRED, {
|
|
212
|
+
expiredAt: new Date(Date.now() - 3600000).toISOString(),
|
|
213
|
+
});
|
|
214
|
+
} catch (error) {
|
|
215
|
+
console.log(formatErrorMessage(error));
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// ============================================================================
|
|
220
|
+
// Example 11: Custom Error Handling in Try-Catch
|
|
221
|
+
// ============================================================================
|
|
222
|
+
|
|
223
|
+
async function asyncErrorHandlingExample() {
|
|
224
|
+
console.log('\n=== Example 11: Async Error Handling ===\n');
|
|
225
|
+
|
|
226
|
+
async function loadPlugin(name) {
|
|
227
|
+
// Simulate plugin loading
|
|
228
|
+
throw new Error(`Cannot find module '${name}'`);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
try {
|
|
232
|
+
await loadPlugin('@rag-pipeline/embedder-openai');
|
|
233
|
+
} catch (error) {
|
|
234
|
+
const enhanced = wrapError(error, ERROR_CODES.PLUGIN_LOAD_FAILED, {
|
|
235
|
+
plugin: '@rag-pipeline/embedder-openai',
|
|
236
|
+
path: './node_modules/@rag-pipeline/embedder-openai',
|
|
237
|
+
reason: error.message,
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
console.log(formatErrorMessage(enhanced));
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ============================================================================
|
|
245
|
+
// Example 12: Error Serialization for Logging
|
|
246
|
+
// ============================================================================
|
|
247
|
+
|
|
248
|
+
function serializationExample() {
|
|
249
|
+
console.log('\n=== Example 12: Error Serialization ===\n');
|
|
250
|
+
|
|
251
|
+
try {
|
|
252
|
+
throw createError(ERROR_CODES.PLUGIN_NOT_FOUND, {
|
|
253
|
+
plugin: 'my-plugin',
|
|
254
|
+
path: './plugins/my-plugin.js',
|
|
255
|
+
});
|
|
256
|
+
} catch (error) {
|
|
257
|
+
// Serialize for logging to external service
|
|
258
|
+
const serialized = error.toJSON();
|
|
259
|
+
console.log(JSON.stringify(serialized, null, 2));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// ============================================================================
|
|
264
|
+
// Example 13: Custom Error Formatting
|
|
265
|
+
// ============================================================================
|
|
266
|
+
|
|
267
|
+
function customFormattingExample() {
|
|
268
|
+
console.log('\n=== Example 13: Custom Formatting ===\n');
|
|
269
|
+
|
|
270
|
+
try {
|
|
271
|
+
throw createError(ERROR_CODES.CONFIG_VALIDATION_FAILED, {
|
|
272
|
+
reason: 'Invalid schema',
|
|
273
|
+
requiredFields: 'metadata.name, pipeline.stages',
|
|
274
|
+
});
|
|
275
|
+
} catch (error) {
|
|
276
|
+
// Minimal format for CI logs
|
|
277
|
+
console.log('CI Format:');
|
|
278
|
+
console.log(
|
|
279
|
+
formatErrorMessage(error, {
|
|
280
|
+
includeCode: true,
|
|
281
|
+
includeSuggestions: false,
|
|
282
|
+
includeDocumentation: false,
|
|
283
|
+
includeStack: false,
|
|
284
|
+
}),
|
|
285
|
+
);
|
|
286
|
+
|
|
287
|
+
console.log('\n---\n');
|
|
288
|
+
|
|
289
|
+
// Detailed format for debugging
|
|
290
|
+
console.log('Debug Format:');
|
|
291
|
+
console.log(
|
|
292
|
+
formatErrorMessage(error, {
|
|
293
|
+
includeCode: true,
|
|
294
|
+
includeSuggestions: true,
|
|
295
|
+
includeDocumentation: true,
|
|
296
|
+
includeContext: true,
|
|
297
|
+
includeStack: true,
|
|
298
|
+
}),
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// ============================================================================
|
|
304
|
+
// Run Examples
|
|
305
|
+
// ============================================================================
|
|
306
|
+
|
|
307
|
+
async function runAllExamples() {
|
|
308
|
+
console.log('='.repeat(70));
|
|
309
|
+
console.log('Error Formatter Usage Examples');
|
|
310
|
+
console.log('='.repeat(70));
|
|
311
|
+
|
|
312
|
+
basicErrorExample();
|
|
313
|
+
richContextExample();
|
|
314
|
+
wrapErrorExample();
|
|
315
|
+
configErrorExample();
|
|
316
|
+
pluginContractExample();
|
|
317
|
+
pipelineErrorExample();
|
|
318
|
+
validationErrorExample();
|
|
319
|
+
hotReloadErrorExample();
|
|
320
|
+
dagCycleExample();
|
|
321
|
+
authErrorExample();
|
|
322
|
+
await asyncErrorHandlingExample();
|
|
323
|
+
serializationExample();
|
|
324
|
+
customFormattingExample();
|
|
325
|
+
|
|
326
|
+
console.log('\n' + '='.repeat(70));
|
|
327
|
+
console.log('Examples Complete');
|
|
328
|
+
console.log('='.repeat(70) + '\n');
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// Run if executed directly
|
|
332
|
+
if (require.main === module) {
|
|
333
|
+
runAllExamples().catch(console.error);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
module.exports = {
|
|
337
|
+
basicErrorExample,
|
|
338
|
+
richContextExample,
|
|
339
|
+
wrapErrorExample,
|
|
340
|
+
configErrorExample,
|
|
341
|
+
pluginContractExample,
|
|
342
|
+
pipelineErrorExample,
|
|
343
|
+
validationErrorExample,
|
|
344
|
+
hotReloadErrorExample,
|
|
345
|
+
dagCycleExample,
|
|
346
|
+
authErrorExample,
|
|
347
|
+
asyncErrorHandlingExample,
|
|
348
|
+
serializationExample,
|
|
349
|
+
customFormattingExample,
|
|
350
|
+
runAllExamples,
|
|
351
|
+
};
|