@devilsdev/rag-pipeline-utils 2.1.12 โ†’ 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.ragrc.schema.json +218 -0
  2. package/README.md +292 -32
  3. package/bin/cli.js +8 -14
  4. package/dist/index.d.ts +237 -0
  5. package/dist/index.mjs +22 -7
  6. package/package.json +17 -13
  7. package/src/ai/adaptive-retrieval.js +191 -0
  8. package/src/ai/federated-learning.js +214 -0
  9. package/src/ai/federation/federated-learning-coordinator.js +345 -0
  10. package/src/ai/federation.js +76 -0
  11. package/src/ai/index-fixed.js +509 -0
  12. package/src/ai/index.js +33 -0
  13. package/src/ai/model-trainer.js +217 -0
  14. package/src/ai/model-training.js +205 -0
  15. package/src/ai/module-template.js +37 -0
  16. package/src/ai/multimodal/multi-modal-processor.js +345 -0
  17. package/src/ai/multimodal-processing.js +268 -0
  18. package/src/ai/multimodal.js +71 -0
  19. package/src/ai/orchestrator.js +70 -0
  20. package/src/ai/retrieval/adaptive-retrieval-engine.js +398 -0
  21. package/src/ai/retrieval-engine.js +69 -0
  22. package/src/ai/training/model-training-orchestrator.js +228 -0
  23. package/src/ci/workflow-analyzer.js +71 -0
  24. package/src/cli/commands/ai-ml.js +967 -0
  25. package/src/cli/commands/docs.js +228 -0
  26. package/src/cli/commands/dx.js +538 -0
  27. package/src/cli/commands/plugin-hub.js +875 -0
  28. package/src/cli/doctor-command.js +639 -0
  29. package/src/cli/enhanced-cli-commands.js +945 -0
  30. package/src/cli/interactive-wizard.js +997 -0
  31. package/src/cli/plugin-marketplace-commands.js +869 -0
  32. package/src/cli.js +17 -0
  33. package/src/config/enhanced-ragrc-schema.js +632 -0
  34. package/src/config/load-config.js +54 -0
  35. package/src/config/load-plugin-config.js +69 -0
  36. package/src/config/normalize-config.js +105 -0
  37. package/src/config/schema-validator.js +315 -0
  38. package/src/config/validate-plugin-schema.js +35 -0
  39. package/src/config/validate-schema.js +113 -0
  40. package/src/contracts/plugin-contracts.js +33 -0
  41. package/src/core/create-pipeline.js +99 -0
  42. package/src/core/dependency-injection.js +242 -0
  43. package/src/core/observability/event-logger.js +241 -0
  44. package/src/core/observability/instrumented-pipeline.js +609 -0
  45. package/src/core/observability/metrics.js +497 -0
  46. package/src/core/observability/tracing.js +781 -0
  47. package/src/core/performance/benchmark.js +516 -0
  48. package/src/core/performance/parallel-processor.js +240 -0
  49. package/src/core/performance/streaming-safeguards.js +611 -0
  50. package/src/core/pipeline-factory.js +60 -0
  51. package/src/core/plugin-contracts.js +89 -0
  52. package/src/core/plugin-marketplace/plugin-metadata.js +485 -0
  53. package/src/core/plugin-marketplace/plugin-publisher.js +574 -0
  54. package/src/core/plugin-marketplace/plugin-registry-format.js +419 -0
  55. package/src/core/plugin-marketplace/version-resolver.js +61 -0
  56. package/src/core/plugin-registry.js +265 -0
  57. package/src/dag/dag-engine.js +954 -0
  58. package/src/dag/dag-node.js +76 -0
  59. package/src/dag/enhanced-dag-engine.js +822 -0
  60. package/src/dx/index.js +40 -0
  61. package/src/dx/integration-templates.js +665 -0
  62. package/src/dx/performance-profiler.js +968 -0
  63. package/src/dx/realtime-debugger.js +755 -0
  64. package/src/dx/visual-pipeline-builder.js +578 -0
  65. package/src/ecosystem/plugin-analytics-dashboard.js +778 -0
  66. package/src/ecosystem/plugin-certification.js +580 -0
  67. package/src/ecosystem/plugin-hub.js +821 -0
  68. package/src/enterprise/audit-logging.js +967 -0
  69. package/src/enterprise/data-governance.js +408 -0
  70. package/src/enterprise/multi-tenancy.js +496 -0
  71. package/src/enterprise/sso-integration.js +903 -0
  72. package/src/evaluate/evaluator.js +52 -0
  73. package/src/evaluate/scoring.js +91 -0
  74. package/src/index.js +67 -0
  75. package/src/ingest.js +59 -0
  76. package/src/loader/csv-loader.js +38 -0
  77. package/src/loader/directory-loader.js +48 -0
  78. package/src/loader/html-loader.js +62 -0
  79. package/src/loader/markdown-loader.js +67 -0
  80. package/src/mocks/openai-embedder.js +119 -0
  81. package/src/mocks/openai-llm.js +77 -0
  82. package/src/mocks/pdf-loader.js +21 -0
  83. package/src/mocks/pinecone-retriever.js +106 -0
  84. package/src/observability/audit-logger.js +758 -0
  85. package/src/observability/dashboard-config.json +675 -0
  86. package/src/observability/event-logger.js +310 -0
  87. package/src/observability/metrics.js +691 -0
  88. package/src/observability/security-monitor.js +830 -0
  89. package/src/observability/slo-monitor.js +319 -0
  90. package/src/observability/telemetry.js +686 -0
  91. package/src/pipeline/pipeline.js +18 -0
  92. package/src/plugins/sample-embedder.js +53 -0
  93. package/src/plugins/sample-llm.js +30 -0
  94. package/src/plugins/sample-loader.js +24 -0
  95. package/src/plugins/sample-retriever.js +84 -0
  96. package/src/query.js +117 -0
  97. package/src/reranker/llm-reranker.js +85 -0
  98. package/src/scripts/script-utilities.js +36 -0
  99. package/src/security/plugin-sandbox.js +586 -0
  100. package/src/security/plugin-signature-verifier.js +414 -0
  101. package/src/security/validator.js +36 -0
  102. package/src/utils/ci/diagnostic-reporter.js +57 -0
  103. package/src/utils/logger.js +211 -0
  104. package/src/utils/performance-monitor.js +561 -0
  105. package/src/utils/plugin-scaffolder.js +447 -0
  106. package/src/utils/retry-policy.js +443 -0
  107. package/src/utils/retry.js +107 -0
  108. package/src/utils/safe-expression-evaluator.js +579 -0
  109. package/src/utils/structured-logger.js +361 -0
  110. package/src/utils/validate-plugin-contract.js +42 -0
@@ -0,0 +1,228 @@
1
+ /**
2
+ * Documentation Automation CLI Commands
3
+ * Comprehensive CLI interface for JSDoc automation, example synchronization, and interactive documentation
4
+ */
5
+
6
+ const { Command } = require('commander');
7
+ // eslint-disable-line global-require
8
+ const { JSDocAutomation } = require('../../docs/jsdoc-automation.js');
9
+ // eslint-disable-line global-require
10
+ const {
11
+ ExampleSynchronization,
12
+ } = require('../../docs/example-synchronization.js');
13
+ // eslint-disable-line global-require
14
+ const {
15
+ InteractiveDocumentation,
16
+ } = require('../../docs/interactive-documentation.js');
17
+ // eslint-disable-line global-require
18
+ const { logger } = require('../../utils/logger.js');
19
+ // eslint-disable-line global-require
20
+
21
+ /**
22
+ * Create documentation automation commands
23
+ */
24
+ function createDocsCommands() {
25
+ const docs = new Command('docs');
26
+ docs.description('Documentation automation and generation commands');
27
+
28
+ // Generate API documentation
29
+ docs
30
+ .command('generate')
31
+ .description('Generate comprehensive API documentation')
32
+ .option('-s, --source <dir>', 'Source directory to scan', 'src')
33
+ .option(
34
+ '-o, --output <dir>',
35
+ 'Output directory for documentation',
36
+ 'docs/api',
37
+ )
38
+ .option(
39
+ '-c, --_config <file>',
40
+ 'JSDoc configuration file',
41
+ 'jsdoc._config.json',
42
+ )
43
+ .option('--no-interactive', 'Skip interactive documentation generation')
44
+ .option('--no-examples', 'Skip example synchronization')
45
+ .option('--include-private', 'Include private APIs in documentation')
46
+ .action(async (_options) => {
47
+ try {
48
+ logger.info('Starting API documentation generation...');
49
+
50
+ const automation = new JSDocAutomation({
51
+ sourceDir: _options.source,
52
+ outputDir: _options.output,
53
+ configFile: _options._config,
54
+ generateInteractive: _options.interactive,
55
+ syncExamples: _options.examples,
56
+ includePrivate: _options.includePrivate,
57
+ });
58
+
59
+ const result = await automation.generateAPIDocumentation();
60
+
61
+ logger.info('API documentation generation completed successfully');
62
+ // eslint-disable-next-line no-console
63
+ console.log('\n๐Ÿ“š Documentation Generation Results:');
64
+ // eslint-disable-line no-console
65
+ // eslint-disable-next-line no-console
66
+ console.log(`โœ… Output Directory: ${result.outputDir}`);
67
+ // eslint-disable-line no-console
68
+ // eslint-disable-next-line no-console
69
+ console.log(`๐Ÿ“„ Generated Files: ${result.generatedFiles.length}`);
70
+ // eslint-disable-line no-console
71
+ // eslint-disable-next-line no-console
72
+ console.log(`๐Ÿ—๏ธ API Structures: ${result.apiStructure.length}`);
73
+ // eslint-disable-line no-console
74
+ // eslint-disable-next-line no-console
75
+ console.log(`๐Ÿ’ก Examples: ${result.examples.length}`);
76
+ // eslint-disable-line no-console
77
+
78
+ if (result.generatedFiles.length > 0) {
79
+ // eslint-disable-next-line no-console
80
+ console.log('\n๐Ÿ“ Generated Files:');
81
+ // eslint-disable-line no-console
82
+ result.generatedFiles.forEach((file) => {
83
+ // eslint-disable-next-line no-console
84
+ console.log(` ${file}`);
85
+ // eslint-disable-line no-console
86
+ });
87
+ }
88
+ } catch (error) {
89
+ logger.error('API documentation generation failed:', error);
90
+ process.exit(1);
91
+ }
92
+ });
93
+
94
+ // Synchronize examples from tests
95
+ docs
96
+ .command('sync-examples')
97
+ .description('Synchronize examples from passing tests')
98
+ .option('-t, --test-dir <dir>', 'Test directory to scan', '__tests__')
99
+ .option(
100
+ '-o, --output <dir>',
101
+ 'Output directory for examples',
102
+ 'docs/examples',
103
+ )
104
+ .option('--no-passing-only', 'Include all tests, not just passing ones')
105
+ .option('--no-runnable', 'Skip generating runnable example files')
106
+ .option('--include-output', 'Include test output in examples')
107
+ .action(async (_options) => {
108
+ try {
109
+ logger.info('Starting example synchronization...');
110
+
111
+ const sync = new ExampleSynchronization({
112
+ testDir: _options.testDir,
113
+ outputDir: _options.output,
114
+ syncOnlyPassing: _options.passingOnly,
115
+ generateRunnable: _options.runnable,
116
+ includeTestOutput: _options.includeOutput,
117
+ });
118
+
119
+ const result = await sync.synchronizeExamples();
120
+
121
+ logger.info('Example synchronization completed successfully');
122
+ // eslint-disable-next-line no-console
123
+ console.log('\n๐Ÿ’ก Example Synchronization Results:');
124
+ // eslint-disable-line no-console
125
+ // eslint-disable-next-line no-console
126
+ console.log(`โœ… Output Directory: ${result.outputDir}`);
127
+ // eslint-disable-line no-console
128
+ // eslint-disable-next-line no-console
129
+ console.log(`๐Ÿ“ Synchronized Examples: ${result.syncedExamples}`);
130
+ // eslint-disable-line no-console
131
+ // eslint-disable-next-line no-console
132
+ console.log(`๐Ÿงช Test Files Processed: ${result.processedTests}`);
133
+ // eslint-disable-line no-console
134
+
135
+ if (result.examples.length > 0) {
136
+ // eslint-disable-next-line no-console
137
+ console.log('\n๐Ÿ“Š Examples by Category:');
138
+ // eslint-disable-line no-console
139
+ result.examples.forEach(([category, examples]) => {
140
+ // eslint-disable-next-line no-console
141
+ console.log(` ${category}: ${examples.length} examples`);
142
+ // eslint-disable-line no-console
143
+ });
144
+ }
145
+ } catch (error) {
146
+ logger.error('Example synchronization failed:', error);
147
+ process.exit(1);
148
+ }
149
+ });
150
+
151
+ // Start interactive documentation server
152
+ docs
153
+ .command('serve')
154
+ .description('Start interactive documentation server')
155
+ .option('-p, --port <number>', 'Server port', '3000')
156
+ .option('-h, --host <host>', 'Server host', 'localhost')
157
+ .option('-d, --docs-dir <dir>', 'Documentation directory', 'docs/api')
158
+ .option('--no-live-reload', 'Disable live reload functionality')
159
+ .option('--enable-code-execution', 'Enable code execution (security risk)')
160
+ .action(async (_options) => {
161
+ try {
162
+ logger.info('Starting interactive documentation server...');
163
+
164
+ const server = new InteractiveDocumentation({
165
+ port: parseInt(_options.port),
166
+ host: _options.host,
167
+ docsDir: _options.docsDir,
168
+ enableLiveReload: _options.liveReload,
169
+ enableCodeExecution: _options.enableCodeExecution,
170
+ });
171
+
172
+ const result = await server.startServer();
173
+
174
+ // eslint-disable-next-line no-console
175
+ console.log('\n๐Ÿš€ Interactive Documentation Server Started!');
176
+ // eslint-disable-line no-console
177
+ // eslint-disable-next-line no-console
178
+ console.log(`๐Ÿ“ URL: ${result.url}`);
179
+ // eslint-disable-line no-console
180
+ // eslint-disable-next-line no-console
181
+ console.log(`๐Ÿ”ง Port: ${result.port}`);
182
+ // eslint-disable-line no-console
183
+ // eslint-disable-next-line no-console
184
+ console.log(`๐Ÿ  Host: ${result.host}`);
185
+ // eslint-disable-line no-console
186
+ // eslint-disable-next-line no-console
187
+ console.log(
188
+ `๐Ÿ”„ Live Reload: ${_options.liveReload ? 'Enabled' : 'Disabled'}`,
189
+ );
190
+ // eslint-disable-line no-console
191
+ // eslint-disable-next-line no-console
192
+ console.log(
193
+ `โšก Code Execution: ${_options.enableCodeExecution ? 'Enabled' : 'Disabled'}`,
194
+ );
195
+ // eslint-disable-line no-console
196
+ // eslint-disable-next-line no-console
197
+ console.log('\nPress Ctrl+C to stop the server');
198
+ // eslint-disable-line no-console
199
+
200
+ // Handle graceful shutdown
201
+ process.on('SIGINT', async () => {
202
+ // eslint-disable-next-line no-console
203
+ console.log('\n๐Ÿ›‘ Shutting down server...');
204
+ // eslint-disable-line no-console
205
+ await server.stopServer();
206
+ process.exit(0);
207
+ });
208
+
209
+ process.on('SIGTERM', async () => {
210
+ // eslint-disable-next-line no-console
211
+ console.log('\n๐Ÿ›‘ Shutting down server...');
212
+ // eslint-disable-line no-console
213
+ await server.stopServer();
214
+ process.exit(0);
215
+ });
216
+ } catch (error) {
217
+ logger.error(
218
+ 'Failed to start interactive documentation server:',
219
+ error,
220
+ );
221
+ process.exit(1);
222
+ }
223
+ });
224
+
225
+ return docs;
226
+ }
227
+
228
+ module.exports = { createDocsCommands };